Repository: kubernetes/kubernetes
File: test/integration/etcd/etcd_storage_path_test.go (Go)

Code before:

// TestEtcdStoragePath tests to make sure that all objects are stored in an expected location in etcd.
// It will start failing when a new type is added to ensure that all future types are added to this test.
// It will also fail when a type gets moved to a different location. Be very careful in this situation because
// it essentially means that you will be break old clusters unless you create some migration path for the old data.
func TestEtcdStoragePath(t *testing.T) {
	// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
	t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")

	featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
	featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)

	apiServer := StartRealAPIServerOrDie(t)
	defer apiServer.Cleanup()
	defer dumpEtcdKVOnFailure(t, apiServer.KV)

	client := &allClient{dynamicClient: apiServer.Dynamic}

	if _, err := apiServer.Client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}, metav1.CreateOptions{}); err != nil {
		t.Fatal(err)
	}

	etcdStorageData := GetEtcdStorageData()

	kindSeen := sets.NewString()
	pathSeen := map[string][]schema.GroupVersionResource{}
	etcdSeen := map[schema.GroupVersionResource]empty{}
	cohabitatingResources := map[string]map[schema.GroupVersionKind]empty{}

	for _, resourceToPersist := range apiServer.Resources {
		t.Run(resourceToPersist.Mapping.Resource.String(), func(t *testing.T) {
			mapping := resourceToPersist.Mapping
			gvk := resourceToPersist.Mapping.GroupVersionKind
			gvResource := resourceToPersist.Mapping.Resource
			kind := gvk.Kind

			if kindAllowList.Has(kind) {
				kindSeen.Insert(kind)
				t.Skip("allowlisted")
			}

			etcdSeen[gvResource] = empty{}
			testData, hasTest := etcdStorageData[gvResource]

			if !hasTest {
				t.Fatalf("no test data for %s.  Please add a test for your new type to GetEtcdStorageData().", gvResource)
			}

			if len(testData.ExpectedEtcdPath) == 0 {
				t.Fatalf("empty test data for %s", gvResource)
			}

Review comment (bug):
This presents a problem , and it took me several days to find it but I still can't figure out how to solve it.

If you checkout my PR https://github.com/kubernetes/kubernetes/pull/128971 and apply the following patch

```diff
diff --git a/pkg/controlplane/apiserver/apis.go b/pkg/controlplane/apiserver/apis.go
index 7ed3a58489b..47dc2cf2467 100644
--- a/pkg/controlplane/apiserver/apis.go
+++ b/pkg/controlplane/apiserver/apis.go
@@ -89,6 +89,7 @@ func (s *Server) InstallAPIs(restStorageProviders ...RESTStorageProvider) error
        nonLegacy := []*genericapiserver.APIGroupInfo{}
 
        // used later in the loop to filter the served resource by those that have expired.
+       klog.Infof("DEBUG ---------- effective emulation version %s", s.GenericAPIServer.EffectiveVersion.EmulationVersion().String())
        resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(s.GenericAPIServer.EffectiveVersion.EmulationVersion())
        if err != nil {
                return err
```
and runs the test
```
go test -timeout 600s -run ^TestEtcdStoragePath$ k8s.io/kubernetes/test/integration/etcd -v -count 1 | grep -E 'DEBUG|TestEtcdStoragePath'
```

you can see that the 1.33 fails because it it keeps using the emulated version 1.32 with 1.33 , this causes the APIExpiration thingy to disable the v1 version of the apiserver for servicecidrs and ipaddresses, and the cluster can not start
```
=== RUN   TestEtcdStoragePath/1.33
I0109 00:21:01.588321 2218629 serving.go:380] Generated self-signed cert (/tmp/TestEtcdStoragePath1.3382862738/apiserver.crt, /tmp/TestEtcdStoragePath1.3382862738/apiserver.key)
I0109 00:21:01.590392 2218629 dynamic_serving_content.go:116] "Loaded a new cert/key pair" name="serving-cert::/tmp/TestEtcdStoragePath1.3382862738/apiserver.crt::/tmp/TestEtcdStoragePath1.3382862738/apiserver.key"
I0109 00:21:01.989808 2218629 apf_controller.go:292] NewTestableController "Controller" with serverConcurrencyLimit=600, name=Controller, asFieldManager="api-priority-and-fairness-config-consumer-v1"
I0109 00:21:02.018277 2218629 apis.go:92] DEBUG ---------- effective emulation version 1.32
```

however, if you run the test individually it works fine

```
PASS
{"level":"warn","ts":"2025-01-09T00:24:46.595770Z","caller":"embed/serve.go:160","msg":"stopping insecure grpc server due to error","error":"accept tcp 127.0.0.1:42889: use of closed network connection"}
{"level":"warn","ts":"2025-01-09T00:24:46.595848Z","caller":"embed/serve.go:162","msg":"stopped insecure grpc server due to error","error":"accept tcp 127.0.0.1:42889: use of closed network connection"}
I0109 00:24:51.596565 2223139 etcd.go:158] etcd didn't exit in 5 seconds, killing it
I0109 00:24:51.596601 2223139 etcd.go:164] etcd exit status: signal: terminated
ok      k8s.io/kubernetes/test/integration/etcd 19.938s
```

cc: @liggitt @Jefftree @siyuanfoundation 


Code after:
	gvk("random.numbers.com", "v1", "Integer"): true,
	gvk("custom.fancy.com", "v2", "Pant"):      true,
}

// TestEtcdStoragePath tests to make sure that all objects are stored in an expected location in etcd.
// It will start failing when a new type is added to ensure that all future types are added to this test.
// It will also fail when a type gets moved to a different location. Be very careful in this situation because
// it essentially means that you will be break old clusters unless you create some migration path for the old data.
func TestEtcdStoragePath(t *testing.T) {
	supportedVersions := GetSupportedEmulatedVersions()
	for _, v := range supportedVersions {
		t.Run(v, func(t *testing.T) {
			testEtcdStoragePathWithVersion(t, v)
		})
	}
}

func testEtcdStoragePathWithVersion(t *testing.T, v string) {
	if v == componentbaseversion.DefaultKubeBinaryVersion {
		featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
		featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
	} else {
		// Only test for beta and GA APIs with emulated version.
		featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse(v))
		featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
		registerEffectiveEmulationVersion(t)
	}

	apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
		// Disable alphas when emulating previous versions.
		if v != componentbaseversion.DefaultKubeBinaryVersion {
			opts.Options.APIEnablement.RuntimeConfig["api/alpha"] = "false"
		}
	})

	defer apiServer.Cleanup()
	defer dumpEtcdKVOnFailure(t, apiServer.KV)

	client := &allClient{dynamicClient: apiServer.Dynamic}

	if _, err := apiServer.Client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}}, metav1.CreateOptions{}); err != nil {
		t.Fatal(err)
	}

	var etcdStorageData map[schema.GroupVersionResource]StorageData
	if v == componentbaseversion.DefaultKubeBinaryVersion {
		etcdStorageData = GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", v, false)
	} else {
		// Drop alphas from etcd data fixtures when emulating previous versions
		// as alphas are not supported with emulation.
		etcdStorageData = GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", v, true)