Change:
This is the tracking issue of promoting CRD validation rules to GA: Listed the blocking issues/features below: [x] [x] [x] [x] Issues be aware of but should not be a GA blocker [ ] [ ] [ ] [ ] [ ] /sig api-machinery /triage accepted
/assign
/cc

Discussion:
genericfeatures.ValidatingAdmissionPolicy: {Default: false, PreRelease: featuregate.Beta}, <del> genericfeatures.CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.Beta}, </del> <ins> genericfeatures.CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31 </ins> genericfeatures.OpenAPIEnums: {Default: true, PreRelease: featuregate.Beta},

"k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" <del> "k8s.io/apiserver/pkg/features" </del> "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" <del> utilfeature "k8s.io/apiserver/pkg/util/feature" </del> ) // strategy implements behavior for CustomResources.

break } } <del> dropDisabledFields(crd, nil) </del> } // PrepareForUpdate clears fields that are not allowed to be set by end users on update.

break } } <del> dropDisabledFields(newCRD, oldCRD) </del> } // Validate validates a new CustomResourceDefinition.

func CustomResourceDefinitionToSelectableFields(obj *apiextensions.CustomResourceDefinition) fields.Set { return generic.ObjectMetaFieldsSet(&obj.ObjectMeta, true) } <del> // dropDisabledFields drops disabled fields that are not used if their associated feature gates // are not enabled. func dropDisabledFields(newCRD *apiextensions.CustomResourceDefinition, oldCRD *apiextensions.CustomResourceDefinition) { if !utilfeature.DefaultFeatureGate.Enabled(features.CustomResourceValidationExpressions) && (oldCRD == nil || (oldCRD != nil && !specHasXValidations(&oldCRD.Spec))) { if newCRD.Spec.Validation != nil { dropXValidationsField(newCRD.Spec.Validation.OpenAPIV3Schema) } for _, v := range newCRD.Spec.Versions { if v.Schema != nil { dropXValidationsField(v.Schema.OpenAPIV3Schema) } } } } // dropXValidationsField drops field XValidations from CRD schema func dropXValidationsField(schema *apiextensions.JSONSchemaProps) { if schema == nil { return } schema.XValidations = nil if schema.AdditionalProperties != nil { dropXValidationsField(schema.AdditionalProperties.Schema) } for def, jsonSchema := range schema.Properties { dropXValidationsField(&jsonSchema) schema.Properties[def] = jsonSchema } if schema.Items != nil { dropXValidationsField(schema.Items.Schema) for i, jsonSchema := range schema.Items.JSONSchemas { dropXValidationsField(&jsonSchema) schema.Items.JSONSchemas[i] = jsonSchema } } for def, jsonSchemaPropsOrStringArray := range schema.Dependencies { dropXValidationsField(jsonSchemaPropsOrStringArray.Schema) schema.Dependencies[def] = jsonSchemaPropsOrStringArray } } func specHasXValidations(spec *apiextensions.CustomResourceDefinitionSpec) bool { return validation.HasSchemaWith(spec, schemaHasXValidations) } func schemaHasXValidations(s *apiextensions.JSONSchemaProps) bool { return validation.SchemaHas(s, func(s *apiextensions.JSONSchemaProps) bool { return s.XValidations != nil }) } </del>

"context" "testing" <del> "github.com/google/go-cmp/cmp" </del> "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" <del> "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" </del> "k8s.io/utils/pointer" )

}) } } <del> // TestDropDisabledFields tests if the drop functionality is working fine or not with feature gate switch func TestDropDisabledFields(t *testing.T) { testCases := []struct { name               string enableXValidations bool crd                *apiextensions.CustomResourceDefinition oldCRD             *apiextensions.CustomResourceDefinition expectedCRD        *apiextensions.CustomResourceDefinition }{ { name:               "For creation, FG disabled, no XValidations, no field drop", enableXValidations: false, crd:                &apiextensions.CustomResourceDefinition{}, oldCRD:             nil, expectedCRD:        &apiextensions.CustomResourceDefinition{}, }, { name:               "For creation, FG disabled, empty XValidations, no field drop", enableXValidations: false, crd: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{}, }, }, oldCRD: nil, expectedCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{}, }, }, }, { name:               "For creation, FG disabled, set XValidations, drop XValidations", enableXValidations: false, crd: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, Dependencies: apiextensions.JSONSchemaDependencies{ "test": apiextensions.JSONSchemaPropsOrStringArray{ Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "size of scoped field should be greater than 0.", }, }, }, }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "subRule": { Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "isTest == true", Message: "isTest should be true.", }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "isTest": { Type: "boolean", }, }, }, }, }, }, }, }, oldCRD: nil, expectedCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", Dependencies: apiextensions.JSONSchemaDependencies{ "test": apiextensions.JSONSchemaPropsOrStringArray{ Schema: &apiextensions.JSONSchemaProps{ Type: "object", }, }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "subRule": { Type: "object", Properties: map[string]apiextensions.JSONSchemaProps{ "isTest": { Type: "boolean", }, }, }, }, }, }, }, }, }, { name:               "For creation, FG enabled, set XValidations, update with XValidations", enableXValidations: true, crd: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, Dependencies: apiextensions.JSONSchemaDependencies{ "test": apiextensions.JSONSchemaPropsOrStringArray{ Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "size of scoped field should be greater than 0.", }, }, }, }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "subRule": { Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "isTest == true", Message: "isTest should be true.", }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "isTest": { Type: "boolean", }, }, }, }, }, }, }, }, oldCRD: nil, expectedCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, Dependencies: apiextensions.JSONSchemaDependencies{ "test": apiextensions.JSONSchemaPropsOrStringArray{ Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "size of scoped field should be greater than 0.", }, }, }, }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "subRule": { Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "isTest == true", Message: "isTest should be true.", }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "isTest": { Type: "boolean", }, }, }, }, }, }, }, }, }, { name:               "For update, FG disabled, oldCRD XValidation in use, don't drop XValidations", enableXValidations: false, crd: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, Dependencies: apiextensions.JSONSchemaDependencies{ "test": apiextensions.JSONSchemaPropsOrStringArray{ Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "size of scoped field should be greater than 0.", }, }, }, }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "subRule": { Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "isTest == true", Message: "isTest should be true.", }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "isTest": { Type: "boolean", }, }, }, }, }, }, }, }, oldCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", Dependencies: apiextensions.JSONSchemaDependencies{ "test": apiextensions.JSONSchemaPropsOrStringArray{ Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "size of scoped field should be greater than 0.", }, }, }, }, }, }, }, }, }, expectedCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, Dependencies: apiextensions.JSONSchemaDependencies{ "test": apiextensions.JSONSchemaPropsOrStringArray{ Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "size of scoped field should be greater than 0.", }, }, }, }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "subRule": { Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "isTest == true", Message: "isTest should be true.", }, }, Properties: map[string]apiextensions.JSONSchemaProps{ "isTest": { Type: "boolean", }, }, }, }, }, }, }, }, }, { name:               "For update, FG disabled, oldCRD has no XValidations, drop XValidations", enableXValidations: false, crd: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, }, }, }, }, oldCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", }, }, }, }, expectedCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", }, }, }, }, }, { name:               "For update, FG enabled, oldCRD has XValidations, updated to newCRD", enableXValidations: true, crd: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, }, }, }, }, oldCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "old data", Message: "old data", }, }, }, }, }, }, expectedCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, }, }, }, }, }, { name:               "For update, FG enabled, oldCRD has no XValidations, updated to newCRD", enableXValidations: true, crd: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, }, }, }, }, oldCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", }, }, }, }, expectedCRD: &apiextensions.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "foos.sigs.k8s.io", Annotations: map[string]string{v1beta1.KubeAPIApprovedAnnotation: "valid"}, ResourceVersion: "1"}, Spec: apiextensions.CustomResourceDefinitionSpec{ Validation: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", XValidations: apiextensions.ValidationRules{ { Rule:    "size(self) > 0", Message: "openAPIV3Schema should contain more than 0 element.", }, }, }, }, }, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CustomResourceValidationExpressions, tc.enableXValidations)() old := tc.oldCRD.DeepCopy() dropDisabledFields(tc.crd, tc.oldCRD) // old crd should never be changed if diff := cmp.Diff(tc.oldCRD, old); diff != "" { t.Fatalf("old crd changed from %v to %v", tc.oldCRD, old) } if diff := cmp.Diff(tc.expectedCRD, tc.crd); diff != "" { t.Fatalf("unexpected crd: %v", tc.crd) } }) } } </del>

// kep: https://kep.k8s.io/2876 // alpha: v1.23 // beta: v1.25 <ins> // stable: v1.29 </ins> // // Enables expression validation for Custom Resource CustomResourceValidationExpressions featuregate.Feature = "CustomResourceValidationExpressions"

ValidatingAdmissionPolicy: {Default: false, PreRelease: featuregate.Beta}, <del> CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.Beta}, </del> <ins> CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31 </ins> EfficientWatchResumption: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},

"k8s.io/kubernetes/test/integration/framework" ) <del> // TestCustomResourceValidatorsWithDisabledFeatureGate test that x-kubernetes-validations work as expected when the // feature gate is disabled. func TestCustomResourceValidatorsWithDisabledFeatureGate(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.CustomResourceValidationExpressions, false)() server, err := apiservertesting.StartTestServer(t, apiservertesting.NewDefaultTestServerOptions(), nil, framework.SharedEtcd()) if err != nil { t.Fatal(err) } defer server.TearDownFn() config := server.ClientConfig apiExtensionClient, err := clientset.NewForConfig(config) if err != nil { t.Fatal(err) } dynamicClient, err := dynamic.NewForConfig(config) if err != nil { t.Fatal(err) } t.Run("x-kubernetes-validations fields MUST be dropped from CRDs that are created when feature gate is disabled", func(t *testing.T) { schemaWithFeatureGateOff := crdWithSchema(t, "WithFeatureGateOff", structuralSchemaWithValidators) crdWithFeatureGateOff, err := fixtures.CreateNewV1CustomResourceDefinition(schemaWithFeatureGateOff, apiExtensionClient, dynamicClient) if err != nil { t.Fatal(err) } s := crdWithFeatureGateOff.Spec.Versions[0].Schema.OpenAPIV3Schema if len(s.XValidations) != 0 { t.Errorf("Expected CRD to have no x-kubernetes-validatons rules but got: %v", s.XValidations) } }) } </del> // TestCustomResourceValidators tests x-kubernetes-validations compile and validate as expected when the feature gate // is enabled. func TestCustomResourceValidators(t *testing.T) {