Repository: kubernetes/kubernetes (language: go)

Issue:
Flagz on kube-apiserver doesn't return parsed flag values
### What happened?

When ComponentFlagz feature is enabled on kube-apiserver, the flag value is not return as expected.
kube-apiserver spec
```yaml
spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=192.168.8.5
    - --feature-gates=ComponentFlagz=true
    - --emulated-version=1.32
    ...
```
The `flagz/` returns
```
~ curl -k --cert /etc/kubernetes/pki/apiserver-kubelet-client.crt --key /etc/kubernetes/pki/apiserver-kubelet-client.key https://localhost:6443/flagz

kube-apiserver flags
Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only.

admission-control:[]
admission-control-config-file:
advertise-address:192.168.8.5
aggregator-reject-forwarding-redirect:true
allow-metric-labels:[]
...
...
egress-selector-config-file:
emulated-version:[]
enable-admission-plugins:[NodeRestriction]
...
...
feature-gates:
...
```

### What did you expect to happen?

The response reflects the correct passed flag value. For example in kube-scheduler
kube-scheduler spec:
```yaml
spec:
  containers:
  - command:
    - kube-scheduler
    - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
    - --feature-gates=ComponentFlagz=true
    - --emulated-version=1.32
   ...
```
The `/flagz` response:
```curl -k --cert /etc/kubernetes/pki/apiserver-kubelet-client.crt --key /etc/kubernetes/pki/apiserver-kubelet-client.key https://localhost:10259/flagz

kube-scheduler flags
Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only.

allow-metric-labels:[]
allow-metric-labels-manifest:
authentication-kubeconfig:/etc/kubernetes/scheduler.conf
...
...
emulated-version:[1.32]
feature-gates::ComponentFlagz=true
...
```

### How can we reproduce it (as minimally and precisely as possible)?

1. Enable the featuregate on kube-apiserver with `--feature-gates=ComponentFlagz=true`
1. request the kube-apiserver endpoint on path `/flagz` and check the response

### Anything else we need to know?

I believe it is due to the flags (instantiated by `s.Flags()`) passed into the options 
https://github.com/kubernetes/kubernetes/blob/925cf7db71c5e36072f99e8b7129523f659ee3a1/cmd/kube-apiserver/app/options/completion.go#L60
is not the same one that actually used by the cmd
https://github.com/kubernetes/kubernetes/blob/925cf7db71c5e36072f99e8b7129523f659ee3a1/cmd/kube-apiserver/app/server.go#L126

### Kubernetes version

<details>

```console
$ kubectl version
Client Version: v1.31.1
Kustomize Version: v5.4.2
Server Version: v1.32.0
```

</details>


### Cloud provider

<details>

</details>


### OS version

<details>

```console
# On Linux:
$ cat /etc/os-release
# paste output here
$ uname -a
# paste output here

# On Windows:
C:\> wmic os get Caption, Version, BuildNumber, OSArchitecture
# paste output here
```

</details>


### Install tools

<details>

</details>


### Container runtime (CRI) and version (if applicable)

<details>

</details>


### Related plugins (CNI, CSI, ...) and versions (if applicable)

<details>

</details>



Discussion / hints:
/sig api-machinery
/sig instrumentation
/cc @richabanker 

Fix patch:
diff --git a/cmd/kube-apiserver/app/options/completion.go b/cmd/kube-apiserver/app/options/completion.go
index 0b83e79a9edd2..89e2dbf9e415d 100644
--- a/cmd/kube-apiserver/app/options/completion.go
+++ b/cmd/kube-apiserver/app/options/completion.go
@@ -57,7 +57,7 @@ func (s *ServerRunOptions) Complete(ctx context.Context) (CompletedOptions, erro
 	if err != nil {
 		return CompletedOptions{}, err
 	}
-	controlplane, err := s.Options.Complete(ctx, s.Flags(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
+	controlplane, err := s.Options.Complete(ctx, []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
 	if err != nil {
 		return CompletedOptions{}, err
 	}
diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go
index 06a8ca81801e5..fea75ca87c662 100644
--- a/cmd/kube-apiserver/app/server.go
+++ b/cmd/kube-apiserver/app/server.go
@@ -49,6 +49,8 @@ import (
 	"k8s.io/component-base/term"
 	utilversion "k8s.io/component-base/version"
 	"k8s.io/component-base/version/verflag"
+	zpagesfeatures "k8s.io/component-base/zpages/features"
+	"k8s.io/component-base/zpages/flagz"
 	"k8s.io/klog/v2"
 	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
@@ -124,6 +126,11 @@ cluster's shared state through which all other components interact.`,
 
 	fs := cmd.Flags()
 	namedFlagSets := s.Flags()
+	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
+		s.Flagz = flagz.NamedFlagSetsReader{
+			FlagSets: namedFlagSets,
+		}
+	}
 	verflag.AddFlags(namedFlagSets.FlagSet("global"))
 	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name(), logs.SkipLoggingConfigurationFlags())
 	options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go
index a6a81ee137d40..a2437a18c0ce2 100644
--- a/pkg/controlplane/apiserver/options/options.go
+++ b/pkg/controlplane/apiserver/options/options.go
@@ -203,7 +203,7 @@ func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
 		"Path to socket where a external JWT signer is listening. This flag is mutually exclusive with --service-account-signing-key-file and --service-account-key-file. Requires enabling feature gate (ExternalServiceAccountTokenSigner)")
 }
 
-func (o *Options) Complete(ctx context.Context, fss cliflag.NamedFlagSets, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
+func (o *Options) Complete(ctx context.Context, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
 	if o == nil {
 		return CompletedOptions{completedOptions: &completedOptions{}}, nil
 	}
@@ -269,8 +269,6 @@ func (o *Options) Complete(ctx context.Context, fss cliflag.NamedFlagSets, alter
 		}
 	}
 
-	completed.Flagz = flagz.NamedFlagSetsReader{FlagSets: fss}
-
 	return CompletedOptions{
 		completedOptions: &completed,
 	}, nil
diff --git a/pkg/controlplane/apiserver/samples/generic/server/server.go b/pkg/controlplane/apiserver/samples/generic/server/server.go
index cbc6d9e563337..be6af3f94d471 100644
--- a/pkg/controlplane/apiserver/samples/generic/server/server.go
+++ b/pkg/controlplane/apiserver/samples/generic/server/server.go
@@ -24,7 +24,6 @@ import (
 	"path/filepath"
 
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -86,7 +85,7 @@ APIs.`,
 
 			ctx := genericapiserver.SetupSignalContext()
 
-			completedOptions, err := s.Complete(ctx, cliflag.NamedFlagSets{FlagSets: map[string]*pflag.FlagSet{"sample_generic_controlplane": fs}}, []string{}, []net.IP{})
+			completedOptions, err := s.Complete(ctx, []string{}, []net.IP{})
 			if err != nil {
 				return err
 			}


Test patch:
diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go
index d7be4ad537eb9..de615a9b1caf5 100644
--- a/cmd/kube-apiserver/app/testing/testserver.go
+++ b/cmd/kube-apiserver/app/testing/testserver.go
@@ -57,6 +57,8 @@ import (
 	basecompatibility "k8s.io/component-base/compatibility"
 	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	logsapi "k8s.io/component-base/logs/api/v1"
+	zpagesfeatures "k8s.io/component-base/zpages/features"
+	"k8s.io/component-base/zpages/flagz"
 	"k8s.io/klog/v2"
 	"k8s.io/kube-aggregator/pkg/apiserver"
 	"k8s.io/kubernetes/pkg/features"
@@ -207,7 +209,8 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 		s.GenericServerRunOptions.RequestTimeout = instanceOptions.RequestTimeout
 	}
 
-	for _, f := range s.Flags().FlagSets {
+	namedFlagSets := s.Flags()
+	for _, f := range namedFlagSets.FlagSets {
 		fs.AddFlagSet(f)
 	}
 
@@ -341,6 +344,9 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 	if err := fs.Parse(customFlags); err != nil {
 		return result, err
 	}
+	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
+		s.Flagz = flagz.NamedFlagSetsReader{FlagSets: namedFlagSets}
+	}
 
 	// the RequestHeader options pointer gets replaced in the case of EnableCertAuth override
 	// and so flags are connected to a struct that no longer appears in the ServerOptions struct
diff --git a/pkg/controlplane/apiserver/config_test.go b/pkg/controlplane/apiserver/config_test.go
index 9f624cbcba743..b6d4f9799273d 100644
--- a/pkg/controlplane/apiserver/config_test.go
+++ b/pkg/controlplane/apiserver/config_test.go
@@ -25,7 +25,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
-	cliflag "k8s.io/component-base/cli/flag"
 	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -47,7 +46,7 @@ func TestBuildGenericConfig(t *testing.T) {
 	s.BindPort = ln.Addr().(*net.TCPAddr).Port
 	opts.SecureServing = s
 
-	completedOptions, err := opts.Complete(context.TODO(), cliflag.NamedFlagSets{}, nil, nil)
+	completedOptions, err := opts.Complete(context.TODO(), nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to complete apiserver options: %v", err)
 	}
diff --git a/pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go b/pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go
index f7dc54535894b..16a0667711b00 100644
--- a/pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go
+++ b/pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go
@@ -37,10 +37,13 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	cliflag "k8s.io/component-base/cli/flag"
 	logsapi "k8s.io/component-base/logs/api/v1"
+	zpagesfeatures "k8s.io/component-base/zpages/features"
+	"k8s.io/component-base/zpages/flagz"
 	"k8s.io/klog/v2"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/test/utils/ktesting"
@@ -126,6 +129,9 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 	o := server.NewOptions()
 	var fss cliflag.NamedFlagSets
 	o.AddFlags(&fss)
+	if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
+		o.Flagz = flagz.NamedFlagSetsReader{FlagSets: fss}
+	}
 
 	fs := pflag.NewFlagSet("test", pflag.PanicOnError)
 	for _, f := range fss.FlagSets {
@@ -164,7 +170,7 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
 	o.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
 	o.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
 
-	completedOptions, err := o.Complete(tCtx, fss, nil, nil)
+	completedOptions, err := o.Complete(tCtx, nil, nil)
 	if err != nil {
 		return result, fmt.Errorf("failed to set default ServerRunOptions: %w", err)
 	}
diff --git a/test/integration/controlplane/kube_apiserver_test.go b/test/integration/controlplane/kube_apiserver_test.go
index 0585f5d4cbe40..8470fe3f79050 100644
--- a/test/integration/controlplane/kube_apiserver_test.go
+++ b/test/integration/controlplane/kube_apiserver_test.go
@@ -132,7 +132,8 @@ func TestLivezAndReadyz(t *testing.T) {
 
 func TestFlagz(t *testing.T) {
 	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ComponentFlagz, true)
-	server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd())
+	testServerFlags := append(framework.DefaultTestServerFlags(), "--emulated-version=1.32")
+	server := kubeapiservertesting.StartTestServerOrDie(t, nil, testServerFlags, framework.SharedEtcd())
 	defer server.TearDownFn()
 
 	client, err := kubernetes.NewForConfig(server.ClientConfig)
@@ -158,6 +159,16 @@ Warning: This endpoint is not meant to be machine parseable, has no formatting c
 	if !bytes.HasPrefix(raw, []byte(expectedHeader)) {
 		t.Fatalf("Header mismatch!\nExpected:\n%s\n\nGot:\n%s", expectedHeader, string(raw))
 	}
+	found := false
+	for _, line := range strings.Split(string(raw), "\n") {
+		if strings.Contains(line, "emulated-version") && strings.Contains(line, "1.32") {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("Expected flag --emulated-version=[1.32] to be reflected in /flagz output, got:\n%s", string(raw))
+	}
 }
 
 // TestOpenAPIDelegationChainPlumbing is a smoke test that checks for