Repository: linkerd/linkerd2
Pull request #8483 (2022-05-14 16:42:42)

Update to kube v0.72, kubert v0.7
Signed-off-by: Oliver Gould <ver@buoyant.io>

<!--  Thanks for sending a pull request!

If you already have a well-structured git commit message, chances are GitHub
set the title and description of this PR to the git commit message subject and
body, respectively. If so, you may delete these instructions and submit your PR.

If this is your first time, please read our contributor guide:
https://github.com/linkerd/linkerd2/blob/main/CONTRIBUTING.md

The title and description of your Pull Request should match the git commit
subject and body, respectively. Git commit messages are structured as follows:

```
Subject

Problem

Solution

Validation

Fixes #[GitHub issue ID]

DCO Sign off
```

Example git commit message:

```
Introduce Pull Request Template

GitHub's community guidelines recommend a pull request template, the repo was
lacking one.

Introduce a `PULL_REQUEST_TEMPLATE.md` file.

Once merged, the
[Community profile checklist](https://github.com/linkerd/linkerd2/community)
should indicate the repo now provides a pull request template.

Fixes #3321

Signed-off-by: Jane Smith <jane.smith@example.com>
```

Note the git commit message subject becomes the pull request title.

For more details around git commits, see the section on Committing in our
contributor guide:
https://github.com/linkerd/linkerd2/blob/main/CONTRIBUTING.md#committing
-->



Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 5493dcb82..0397bdf43 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -214,9 +214,9 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.13.4"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c"
+checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -224,9 +224,9 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.13.4"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610"
+checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f"
 dependencies = [
  "fnv",
  "ident_case",
@@ -238,9 +238,9 @@ dependencies = [
 
 [[package]]
 name = "darling_macro"
-version = "0.13.4"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835"
+checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5"
 dependencies = [
  "darling_core",
  "quote",
@@ -760,9 +760,9 @@ dependencies = [
 
 [[package]]
 name = "kube"
-version = "0.71.0"
+version = "0.72.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "342744dfeb81fe186b84f485b33f12c6a15d3396987d933b06a566a3db52ca38"
+checksum = "977951a1d7dbce4c6f9c2cbed0711f568df9944010fafa88161a20f7e5163bd3"
 dependencies = [
  "k8s-openapi",
  "kube-client",
@@ -773,9 +773,9 @@ dependencies = [
 
 [[package]]
 name = "kube-client"
-version = "0.71.0"
+version = "0.72.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f69a504997799340408635d6e351afb8aab2c34ca3165e162f41b3b34a69a79"
+checksum = "625e8a89c79dd4e590217838f32274ccbb0f4ecc1db181f84aba3df635fca729"
 dependencies = [
  "base64",
  "bytes",
@@ -798,7 +798,7 @@ dependencies = [
  "pin-project",
  "rand",
  "rustls",
- "rustls-pemfile 0.3.0",
+ "rustls-pemfile",
  "secrecy",
  "serde",
  "serde_json",
@@ -815,9 +815,9 @@ dependencies = [
 
 [[package]]
 name = "kube-core"
-version = "0.71.0"
+version = "0.72.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a247487699941baaf93438d65b12d4e32450bea849d619d19ed394e8a4a645"
+checksum = "0491dcd9adca79a96c63404aa978137f5fe1d3db5dcb6e5c0b073f915dbdd49d"
 dependencies = [
  "chrono",
  "form_urlencoded",
@@ -833,9 +833,9 @@ dependencies = [
 
 [[package]]
 name = "kube-derive"
-version = "0.71.0"
+version = "0.72.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "203f7c5acf9d0dfb0b08d44ec1d66ace3d1dfe0cdd82e65e274f3f96615d666c"
+checksum = "f19faefec284e5eec6680c74cd54d29823f38f0cf8811d54265877e1963996ad"
 dependencies = [
  "darling",
  "proc-macro2",
@@ -846,9 +846,9 @@ dependencies = [
 
 [[package]]
 name = "kube-runtime"
-version = "0.71.0"
+version = "0.72.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02ea50e6ed56578e1d1d02548901b12fe6d3edbf110269a396955e285d487973"
+checksum = "bb67002cbacd6cd99d1b1d0890785a84b6408158972f2561317bc4cdc06c981e"
 dependencies = [
  "ahash",
  "backoff",
@@ -870,9 +870,9 @@ dependencies = [
 
 [[package]]
 name = "kubert"
-version = "0.6.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f55c0a5ecae5b8b65a6982e2e97abb0d58d7fd561a2b298120f89484eff8f352"
+checksum = "26e7528462e3f48d9a5693e7c27ecb6c7e1615829cd6ff542def596afdd9e944"
 dependencies = [
  "ahash",
  "clap",
@@ -885,7 +885,7 @@ dependencies = [
  "kube-runtime",
  "parking_lot",
  "pin-project-lite",
- "rustls-pemfile 1.0.0",
+ "rustls-pemfile",
  "serde",
  "thiserror",
  "tokio",
@@ -1496,20 +1496,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"
 dependencies = [
  "openssl-probe",
- "rustls-pemfile 1.0.0",
+ "rustls-pemfile",
  "schannel",
  "security-framework",
 ]
 
-[[package]]
-name = "rustls-pemfile"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360"
-dependencies = [
- "base64",
-]
-
 [[package]]
 name = "rustls-pemfile"
 version = "1.0.0"
@@ -1995,9 +1986,9 @@ dependencies = [
 
 [[package]]
 name = "tower-http"
-version = "0.2.5"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8"
+checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae"
 dependencies = [
  "base64",
  "bitflags",
diff --git a/cli/cmd/check.go b/cli/cmd/check.go
index e72c43bc0..c349197e4 100644
--- a/cli/cmd/check.go
+++ b/cli/cmd/check.go
@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -134,6 +135,11 @@ func configureAndRunChecks(cmd *cobra.Command, wout io.Writer, werr io.Writer, o
 		healthcheck.LinkerdVersionChecks,
 	}
 
+	crdManifest := bytes.Buffer{}
+	err = renderCRDs(&crdManifest)
+	if err != nil {
+		return err
+	}
 	var installManifest string
 	var values *charts.Values
 	if options.preInstallOnly {
@@ -179,6 +185,7 @@ func configureAndRunChecks(cmd *cobra.Command, wout io.Writer, werr io.Writer, o
 		RetryDeadline:         time.Now().Add(options.wait),
 		CNIEnabled:            options.cniEnabled,
 		InstallManifest:       installManifest,
+		CRDManifest:           crdManifest.String(),
 		ChartValues:           values,
 	})
 
diff --git a/cli/cmd/install.go b/cli/cmd/install.go
index 7ec0e4a30..4a1a65685 100644
--- a/cli/cmd/install.go
+++ b/cli/cmd/install.go
@@ -121,7 +121,13 @@ A full list of configurable values can be found at https://www.github.com/linker
 				}
 
 				if !crds {
-					err = healthcheck.CheckCustomResourceDefinitions(cmd.Context(), k8sAPI, true)
+					crds := bytes.Buffer{}
+					err := renderCRDs(&crds)
+					if err != nil {
+						fmt.Fprintf(os.Stderr, "%q", err)
+						os.Exit(1)
+					}
+					err = healthcheck.CheckCustomResourceDefinitions(cmd.Context(), k8sAPI, crds.String())
 					if err != nil {
 						fmt.Fprintln(os.Stderr, "Linkerd CRDs must be installed first. Run linkerd install with the --crds flag.")
 						os.Exit(1)
diff --git a/cli/cmd/upgrade.go b/cli/cmd/upgrade.go
index 1fb014d18..a32617b5a 100644
--- a/cli/cmd/upgrade.go
+++ b/cli/cmd/upgrade.go
@@ -164,6 +164,18 @@ func makeUpgradeFlags() *pflag.FlagSet {
 }
 
 func upgradeControlPlaneRunE(ctx context.Context, k *k8s.KubernetesAPI, flags []flag.Flag, options valuespkg.Options) error {
+
+	crds := bytes.Buffer{}
+	err := renderCRDs(&crds)
+	if err != nil {
+		return err
+	}
+
+	err = healthcheck.CheckCustomResourceDefinitions(ctx, k, crds.String())
+	if err != nil {
+		return fmt.Errorf("Linkerd CRDs must be installed first. Run linkerd upgrade with the --crds flag:\n%w", err)
+	}
+
 	buf, err := upgradeControlPlane(ctx, k, flags, options)
 	if err != nil {
 		return err
diff --git a/deny.toml b/deny.toml
index aa8fe5f5d..7620fe623 100644
--- a/deny.toml
+++ b/deny.toml
@@ -11,11 +11,7 @@ vulnerability = "deny"
 unmaintained = "warn"
 yanked = "deny"
 notice = "warn"
-ignore = [
-    # https://github.com/kube-rs/kube-rs/issues/650
-    # https://github.com/kube-rs/kube-rs/issues/660
-    "RUSTSEC-2020-0159",
-]
+ignore = []
 
 [licenses]
 unlicensed = "deny"
@@ -46,8 +42,6 @@ deny = []
 skip = [
     # Waiting on h2, tonic, etc
     { name = "tokio-util", version = "0.6" },
-    # Waiting on kube-client
-    { name = "rustls-pemfile", version = "0.3" },
 ]
 
 [sources]
diff --git a/pkg/healthcheck/healthcheck.go b/pkg/healthcheck/healthcheck.go
index ce7fb14e6..204b07e5f 100644
--- a/pkg/healthcheck/healthcheck.go
+++ b/pkg/healthcheck/healthcheck.go
@@ -26,6 +26,8 @@ import (
 	admissionRegistration "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -394,6 +396,7 @@ type Options struct {
 	RetryDeadline         time.Time
 	CNIEnabled            bool
 	InstallManifest       string
+	CRDManifest           string
 	ChartValues           *l5dcharts.Values
 }
 
@@ -769,7 +772,7 @@ func (hc *HealthChecker) allCategories() []*Category {
 					hintAnchor:  "l5d-existence-crd",
 					fatal:       true,
 					check: func(ctx context.Context) error {
-						return CheckCustomResourceDefinitions(ctx, hc.kubeAPI, true)
+						return CheckCustomResourceDefinitions(ctx, hc.kubeAPI, hc.CRDManifest)
 					},
 				},
 				{
@@ -2060,29 +2063,56 @@ func (hc *HealthChecker) checkValidatingWebhookConfigurations(ctx context.Contex
 
 // CheckCustomResourceDefinitions checks that all of the Linkerd CRDs are
 // installed on the cluster.
-func CheckCustomResourceDefinitions(ctx context.Context, k8sAPI *k8s.KubernetesAPI, shouldExist bool) error {
-	options := metav1.ListOptions{
-		LabelSelector: controlPlaneComponentsSelector(),
-	}
-	crdList, err := k8sAPI.Apiextensions.ApiextensionsV1().CustomResourceDefinitions().List(ctx, options)
-	if err != nil {
-		return err
+func CheckCustomResourceDefinitions(ctx context.Context, k8sAPI *k8s.KubernetesAPI, expectedCRDManifests string) error {
+
+	crdYamls := strings.Split(expectedCRDManifests, "---")
+	crdVersions := []struct{ name, version string }{}
+	for _, crdYaml := range crdYamls {
+		var crd apiextv1.CustomResourceDefinition
+		err := yaml.Unmarshal([]byte(crdYaml), &crd)
+		if err != nil {
+			return err
+		}
+		if len(crd.Spec.Versions) == 0 {
+			continue
+		}
+		versionIndex := len(crd.Spec.Versions) - 1
+		crdVersions = append(crdVersions, struct{ name, version string }{
+			name:    crd.Name,
+			version: crd.Spec.Versions[versionIndex].Name,
+		})
 	}
 
-	objects := []runtime.Object{}
-	for _, item := range crdList.Items {
-		item := item // pin
-		objects = append(objects, &item)
+	errMsgs := []string{}
+
+	for _, crdVersion := range crdVersions {
+		name := crdVersion.name
+		version := crdVersion.version
+
+		crd, err := k8sAPI.Apiextensions.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, name, metav1.GetOptions{})
+		if err != nil && kerrors.IsNotFound(err) {
+			errMsgs = append(errMsgs, fmt.Sprintf("missing %s", name))
+			continue
+		} else if err != nil {
+			return err
+		}
+		if !crdHasVersion(crd, version) {
+			errMsgs = append(errMsgs, fmt.Sprintf("CRD %s is missing version %s", name, version))
+		}
 	}
+	if len(errMsgs) > 0 {
+		return errors.New(strings.Join(errMsgs, ", "))
+	}
+	return nil
+}
 
-	return checkResources("CustomResourceDefinitions", objects, []string{
-		"authorizationpolicies.policy.linkerd.io",
-		"meshtlsauthentications.policy.linkerd.io",
-		"networkauthentications.policy.linkerd.io",
-		"servers.policy.linkerd.io",
-		"serverauthorizations.policy.linkerd.io",
-		"serviceprofiles.linkerd.io",
-	}, shouldExist)
+func crdHasVersion(crd *v1.CustomResourceDefinition, version string) bool {
+	for _, crdVersion := range crd.Spec.Versions {
+		if crdVersion.Name == version {
+			return true
+		}
+	}
+	return false
 }
 
 // CheckNodesHaveNonDockerRuntime checks that each node has a non-Docker
diff --git a/policy-controller/Cargo.toml b/policy-controller/Cargo.toml
index 8a31ca323..81c977713 100644
--- a/policy-controller/Cargo.toml
+++ b/policy-controller/Cargo.toml
@@ -31,7 +31,7 @@ tokio-stream = { version = "0.1", features = ["sync"] }
 tracing = "0.1"
 
 [dependencies.kube]
-version = "0.71"
+version = "0.72"
 default-features = false
 features = [
     "admission",
@@ -39,7 +39,7 @@ features = [
 ]
 
 [dependencies.kubert]
-version = "0.6"
+version = "0.7"
 default-features = false
 features = [
     "clap",
diff --git a/policy-controller/k8s/api/Cargo.toml b/policy-controller/k8s/api/Cargo.toml
index ad9f4deb6..35d0166a2 100644
--- a/policy-controller/k8s/api/Cargo.toml
+++ b/policy-controller/k8s/api/Cargo.toml
@@ -7,7 +7,7 @@ publish = false
 
 [dependencies]
 k8s-openapi = { version = "0.14", default-features = false, features = ["v1_20"] }
-kube = { version = "0.71", default-features = false, features = ["client", "derive", "runtime"] }
+kube = { version = "0.72", default-features = false, features = ["client", "derive", "runtime"] }
 ipnet = { version = "2.5", features = ["json"] }
 schemars = "0.8"
 serde = { version = "1", features = ["derive"] }
diff --git a/policy-controller/k8s/index/Cargo.toml b/policy-controller/k8s/index/Cargo.toml
index 260877599..188a5ea97 100644
--- a/policy-controller/k8s/index/Cargo.toml
+++ b/policy-controller/k8s/index/Cargo.toml
@@ -9,7 +9,7 @@ publish = false
 ahash = "0.7"
 anyhow = "1"
 futures = { version = "0.3", default-features = false }
-kubert = { version = "0.6", default-features = false, features = ["index"] }
+kubert = { version = "0.7", default-features = false, features = ["index"] }
 linkerd-policy-controller-core = { path = "../../core" }
 linkerd-policy-controller-k8s-api = { path = "../api" }
 parking_lot = "0.12"


Test patch:
diff --git a/pkg/healthcheck/healthcheck_test.go b/pkg/healthcheck/healthcheck_test.go
index 2a1631232..ee1997302 100644
--- a/pkg/healthcheck/healthcheck_test.go
+++ b/pkg/healthcheck/healthcheck_test.go
@@ -550,13 +550,13 @@ metadata:
     linkerd.io/control-plane-ns: test-ns
 `}
 	crds := []string{}
-	for _, name := range []string{
-		"authorizationpolicies.policy.linkerd.io",
-		"meshtlsauthentications.policy.linkerd.io",
-		"networkauthentications.policy.linkerd.io",
-		"serverauthorizations.policy.linkerd.io",
-		"servers.policy.linkerd.io",
-		"serviceprofiles.linkerd.io",
+	for _, crd := range []struct{ name, version string }{
+		{name: "authorizationpolicies.policy.linkerd.io", version: "v1alpha1"},
+		{name: "meshtlsauthentications.policy.linkerd.io", version: "v1alpha1"},
+		{name: "networkauthentications.policy.linkerd.io", version: "v1alpha1"},
+		{name: "serverauthorizations.policy.linkerd.io", version: "v1beta1"},
+		{name: "servers.policy.linkerd.io", version: "v1beta1"},
+		{name: "serviceprofiles.linkerd.io", version: "v1alpha2"},
 	} {
 		crds = append(crds, fmt.Sprintf(`
 apiVersion: apiextensions.k8s.io/v1
@@ -565,7 +565,9 @@ metadata:
   name: %s
   labels:
     linkerd.io/control-plane-ns: test-ns
-`, name))
+spec:
+  versions:
+  - name: %s`, crd.name, crd.version))
 	}
 	mutatingWebhooks := []string{`
 apiVersion: admissionregistration.k8s.io/v1
@@ -622,7 +624,7 @@ metadata:
 				"linkerd-config control plane ClusterRoles exist",
 				"linkerd-config control plane ClusterRoleBindings exist",
 				"linkerd-config control plane ServiceAccounts exist",
-				"linkerd-config control plane CustomResourceDefinitions exist: missing CustomResourceDefinitions: authorizationpolicies.policy.linkerd.io, meshtlsauthentications.policy.linkerd.io, networkauthentications.policy.linkerd.io, serverauthorizations.policy.linkerd.io, servers.policy.linkerd.io, serviceprofiles.linkerd.io",
+				"linkerd-config control plane CustomResourceDefinitions exist: missing authorizationpolicies.policy.linkerd.io, missing meshtlsauthentications.policy.linkerd.io, missing networkauthentications.policy.linkerd.io, missing serverauthorizations.policy.linkerd.io, missing servers.policy.linkerd.io, missing serviceprofiles.linkerd.io",
 			},
 		},
 		{
@@ -690,6 +692,7 @@ metadata:
 				[]CategoryID{LinkerdConfigChecks},
 				&Options{
 					ControlPlaneNamespace: "test-ns",
+					CRDManifest:           strings.Join(crds, "\n---\n"),
 				},
 			)
 
diff --git a/policy-test/Cargo.toml b/policy-test/Cargo.toml
index 18d0592d6..e79bfbe3c 100644
--- a/policy-test/Cargo.toml
+++ b/policy-test/Cargo.toml
@@ -25,7 +25,7 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [dependencies.kube]
-version = "0.71"
+version = "0.72"
 default-features = false
 features = [
     "client",
diff --git a/test/integration/upgrade-edge/upgrade_edge_test.go b/test/integration/upgrade-edge/upgrade_edge_test.go
index 0e2166b18..a602245f1 100644
--- a/test/integration/upgrade-edge/upgrade_edge_test.go
+++ b/test/integration/upgrade-edge/upgrade_edge_test.go
@@ -197,8 +197,22 @@ func TestUpgradeCli(t *testing.T) {
 		"--set", "heartbeatSchedule=1 2 3 4 5",
 		"--set", "proxyInit.ignoreOutboundPorts=1234\\,5678",
 	}
-	exec := append([]string{cmd}, args...)
+
+	// Upgrade CRDs.
+	exec := append([]string{cmd}, append(args, "--crds")...)
 	out, err := TestHelper.LinkerdRun(exec...)
+	if err != nil {
+		testutil.AnnotatedFatal(t, "'linkerd upgrade --crds' command failed", err)
+	}
+	cmdOut, err := TestHelper.KubectlApply(out, "")
+	if err != nil {
+		testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
+			"'kubectl apply' command failed\n%s", cmdOut)
+	}
+
+	// Upgrade control plane.
+	exec = append([]string{cmd}, args...)
+	out, err = TestHelper.LinkerdRun(exec...)
 	if err != nil {
 		testutil.AnnotatedFatal(t, "'linkerd upgrade' command failed", err)
 	}
@@ -207,7 +221,7 @@ func TestUpgradeCli(t *testing.T) {
 	// that we intend to be delete in this stage to prevent it
 	// from deleting other resources that have the
 	// label
-	cmdOut, err := TestHelper.KubectlApplyWithArgs(out, []string{
+	cmdOut, err = TestHelper.KubectlApplyWithArgs(out, []string{
 		"--prune",
 		"-l", "linkerd.io/control-plane-ns=linkerd",
 		"--prune-whitelist", "apps/v1/deployment",