Change:
We are using (and ) to interact with Kubernetes clusters behind a couple of proxy layers. One of these proxies completes the headers to include character encoding details. This was working fine until recently, when client-go/kubectl have started to fail when interacting with the discovery API. It seems that a recent change has made kubectl more restrictive of the content-type that it expects. I suspect that it is related to this commit: For example, this is an error from a kubectl command: We can of course try to fix it on our end, removing that charset information. However, it is a valid content-type and the way kubectl is handling the content-type there seems like it is relying on implementation details of kube-apiserver (that it does not add metadata to the content-type). I propose to fix this issue in , by properly parsing and handling the content-type.
cc /kind regression /kind bug /sig api-machinery /priority important-soon looks like the switch statement in downloadLegacy() needs to be more tolerant of the variety of content-types it can get from a server (including text/plain or no content-type if old servers could have returned that...)
/transfer kubernetes/kubernetes
Something went wrong or the destination repo kubernetes/kubernetes/kubernetes does not exist. <detailsIn response to : /transfer kubernetes/kubernetes Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/transfer kubernetes
Yep, the string comparison for the content-type is flimsy, let's address that. Thanks for the report!
/assign
/triage accepted
/cc
The fix is being cherry-picked back to: : :

Discussion:
"context" "encoding/json" "fmt" <ins> "mime" </ins> "net/http" "net/url" "sort"

defaultBurst = 300 AcceptV1 = runtime.ContentTypeJSON <del> // Aggregated discovery content-type (currently v2beta1). NOTE: Currently, we are assuming the order // for "g", "v", and "as" from the server. We can only compare this string if we can make that assumption. </del> <ins> // Aggregated discovery content-type (v2beta1). NOTE: content-type parameters // MUST be ordered (g, v, as) for server in "Accept" header (BUT we are resilient // to ordering when comparing returned values in "Content-Type" header). </ins> AcceptV2Beta1 = runtime.ContentTypeJSON + ";" + "g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList" // Prioritize aggregated discovery by placing first in the order of discovery accept types. acceptDiscoveryFormats = AcceptV2Beta1 + "," + AcceptV1

var resourcesByGV map[schema.GroupVersion]*metav1.APIResourceList // Switch on content-type server responded with: aggregated or unaggregated. <del> switch responseContentType { case AcceptV1: </del> <ins> switch { case isV2Beta1ContentType(responseContentType): var aggregatedDiscovery apidiscovery.APIGroupDiscoveryList err = json.Unmarshal(body, &aggregatedDiscovery) if err != nil { return nil, nil, nil, err } apiGroupList, resourcesByGV, failedGVs = SplitGroupsAndResources(aggregatedDiscovery) default: // Default is unaggregated discovery v1. </ins> var v metav1.APIVersions err = json.Unmarshal(body, &v) if err != nil {

apiGroup = apiVersionsToAPIGroup(&v) } apiGroupList.Groups = []metav1.APIGroup{apiGroup} <del> case AcceptV2Beta1: var aggregatedDiscovery apidiscovery.APIGroupDiscoveryList err = json.Unmarshal(body, &aggregatedDiscovery) if err != nil { return nil, nil, nil, err } apiGroupList, resourcesByGV, failedGVs = SplitGroupsAndResources(aggregatedDiscovery) default: return nil, nil, nil, fmt.Errorf("Unknown discovery response content-type: %s", responseContentType) </del> } return apiGroupList, resourcesByGV, failedGVs, nil

failedGVs := map[schema.GroupVersion]error{} var resourcesByGV map[schema.GroupVersion]*metav1.APIResourceList // Switch on content-type server responded with: aggregated or unaggregated. <del> switch responseContentType { case AcceptV1: err = json.Unmarshal(body, apiGroupList) if err != nil { return nil, nil, nil, err } case AcceptV2Beta1: </del> <ins> switch { case isV2Beta1ContentType(responseContentType): </ins> var aggregatedDiscovery apidiscovery.APIGroupDiscoveryList err = json.Unmarshal(body, &aggregatedDiscovery) if err != nil {

} apiGroupList, resourcesByGV, failedGVs = SplitGroupsAndResources(aggregatedDiscovery) default: <del> return nil, nil, nil, fmt.Errorf("Unknown discovery response content-type: %s", responseContentType) </del> <ins> // Default is unaggregated discovery v1. err = json.Unmarshal(body, apiGroupList) if err != nil { return nil, nil, nil, err } </ins> } return apiGroupList, resourcesByGV, failedGVs, nil } <ins> // isV2Beta1ContentType checks of the content-type string is both // "application/json" and contains the v2beta1 content-type params. // NOTE: This function is resilient to the ordering of the // content-type parameters, as well as parameters added by // intermediaries such as proxies or gateways. Examples: // //	"application/json; g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList" = true //	"application/json; as=APIGroupDiscoveryList;v=v2beta1;g=apidiscovery.k8s.io" = true //	"application/json; as=APIGroupDiscoveryList;v=v2beta1;g=apidiscovery.k8s.io;charset=utf-8" = true //	"application/json" = false //	"application/json; charset=UTF-8" = false func isV2Beta1ContentType(contentType string) bool { base, params, err := mime.ParseMediaType(contentType) if err != nil { return false } return runtime.ContentTypeJSON == base && params["g"] == "apidiscovery.k8s.io" && params["v"] == "v2beta1" && params["as"] == "APIGroupDiscoveryList" } </ins> // ServerGroups returns the supported groups, with information like supported versions and the // preferred version. func (d *DiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) {

} output, err := json.Marshal(agg) require.NoError(t, err) <del> // Content-type is "aggregated" discovery format. w.Header().Set("Content-Type", AcceptV2Beta1) </del> <ins> // Content-Type is "aggregated" discovery format. Add extra parameter // to ensure we are resilient to these extra parameters. w.Header().Set("Content-Type", AcceptV2Beta1+"; charset=utf-8") </ins> w.WriteHeader(http.StatusOK) w.Write(output) }))

} output, err := json.Marshal(agg) require.NoError(t, err) <del> // Content-type is "aggregated" discovery format. w.Header().Set("Content-Type", AcceptV2Beta1) </del> <ins> // Content-type is "aggregated" discovery format. Add extra parameter // to ensure we are resilient to these extra parameters. w.Header().Set("Content-Type", AcceptV2Beta1+"; charset=utf-8") </ins> w.WriteHeader(status) w.Write(output) }))

} output, err := json.Marshal(agg) require.NoError(t, err) <del> // Content-type is "aggregated" discovery format. w.Header().Set("Content-Type", AcceptV2Beta1) </del> <ins> // Content-type is "aggregated" discovery format. Add extra parameter // to ensure we are resilient to these extra parameters. w.Header().Set("Content-Type", AcceptV2Beta1+"; charset=utf-8") </ins> w.WriteHeader(http.StatusOK) w.Write(output) }))

} } <ins> func TestDiscoveryContentTypeVersion(t *testing.T) { tests := []struct { contentType string isV2Beta1   bool }{ { contentType: "application/json; g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList", isV2Beta1:   true, }, { // content-type parameters are not in correct order, but comparison ignores order. contentType: "application/json; v=v2beta1;as=APIGroupDiscoveryList;g=apidiscovery.k8s.io", isV2Beta1:   true, }, { // content-type parameters are not in correct order, but comparison ignores order. contentType: "application/json; as=APIGroupDiscoveryList;g=apidiscovery.k8s.io;v=v2beta1", isV2Beta1:   true, }, { // Ignores extra parameter "charset=utf-8" contentType: "application/json; g=apidiscovery.k8s.io;v=v2beta1;as=APIGroupDiscoveryList;charset=utf-8", isV2Beta1:   true, }, { contentType: "application/json", isV2Beta1:   false, }, { contentType: "application/json; charset=UTF-8", isV2Beta1:   false, }, { contentType: "text/json", isV2Beta1:   false, }, { contentType: "text/html", isV2Beta1:   false, }, { contentType: "", isV2Beta1:   false, }, } for _, test := range tests { isV2Beta1 := isV2Beta1ContentType(test.contentType) assert.Equal(t, test.isV2Beta1, isV2Beta1) } } </ins> func TestUseLegacyDiscovery(t *testing.T) { // Default client sends aggregated discovery accept format (first) as well as legacy format. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {