Repository: kumahq/kuma
Pull request #4958 (2022-09-01 19:18:21)

feat: add option to install with experimental transparent proxy
Motivation for this was #4442 but it may be superfluous depending on when we want to switch to the new engine

### Checklist prior to review

<!--
Each of these sections need to be filled by the author when opening the PR.

If something doesn't apply please check the box and add a justification after the `--`
-->

- [x] Link to docs PR or issue --
- [x] Link to UI issue or PR --
- [x] Is the [issue worked on linked][1]? --
- [x] The PR does not hardcode values that might break projects that depend on kuma (e.g. "kumahq" as a image registry) --
- [x] The PR will work for both Linux and Windows, system specific functions like `syscall.Mkfifo` have equivalent implementation on the other OS --
- [x] Unit Tests --
- [x] E2E Tests --
- [x] Manual Universal Tests --
- [x] Manual Kubernetes Tests --
- [x] Do you need to update [`UPGRADE.md`](../blob/master/UPGRADE.md)? --
- [x] Does it need to be backported according to the [backporting policy](../blob/master/CONTRIBUTING.md#backporting)? --
- [x] Do you need to explicitly set a [`> Changelog:` entry here](../blob/master/CONTRIBUTING.md#submitting-a-patch)?

[1]: https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword



Patch:
diff --git a/deployments/charts/kuma/README.md b/deployments/charts/kuma/README.md
index b8463490f..1f30deefb 100644
--- a/deployments/charts/kuma/README.md
+++ b/deployments/charts/kuma/README.md
@@ -162,6 +162,7 @@ A Helm chart for the Kuma Control Plane
 | hooks.containerSecurityContext | object | `{}` | Security context at the container level for crd/webhook/ns |
 | experimental.gatewayAPI | bool | `false` | If true, it installs experimental Gateway API support |
 | experimental.cni | bool | `false` | If true, it installs experimental new version of the CNI |
+| experimental.transparentProxy | bool | `false` | If true, use the new transparent proxy engine |
 | experimental.ebpf.enabled | bool | `false` | If true, ebpf will be used instead of using iptables to install/configure transparent proxy |
 | experimental.ebpf.instanceIPEnvVarName | string | `"INSTANCE_IP"` | Name of the environmental variable which will contain the IP address of a pod |
 | experimental.ebpf.bpffsPath | string | `"/run/kuma/bpf"` | Path where BPF file system should be mounted |
diff --git a/deployments/charts/kuma/templates/_helpers.tpl b/deployments/charts/kuma/templates/_helpers.tpl
index 3f52b05bd..002718b85 100644
--- a/deployments/charts/kuma/templates/_helpers.tpl
+++ b/deployments/charts/kuma/templates/_helpers.tpl
@@ -279,6 +279,10 @@ env:
 - name: KUMA_RUNTIME_KUBERNETES_NODE_TAINT_CONTROLLER_CNI_APP
   value: "{{ include "kuma.name" . }}-cni"
 {{- end }}
+{{- if .Values.experimental.transparentProxy }}
+- name: KUMA_RUNTIME_KUBERNETES_INJECTOR_TRANSPARENT_PROXY_V2
+  value: "true"
+{{- end }}
 {{- if .Values.experimental.ebpf.enabled }}
 - name: KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_ENABLED
   value: "true"
diff --git a/deployments/charts/kuma/values.yaml b/deployments/charts/kuma/values.yaml
index 3670279bb..6cc108372 100644
--- a/deployments/charts/kuma/values.yaml
+++ b/deployments/charts/kuma/values.yaml
@@ -668,6 +668,8 @@ experimental:
   gatewayAPI: false
   # -- If true, it installs experimental new version of the CNI
   cni: false
+  # -- If true, use the new transparent proxy engine
+  transparentProxy: false
   # Configuration for the experimental ebpf mode for transparent proxy
   ebpf:
     # -- If true, ebpf will be used instead of using iptables to install/configure transparent proxy
diff --git a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
index 2b26f34c5..7f6100a5c 100644
--- a/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
+++ b/pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
@@ -272,6 +272,7 @@ runtime:
         enabled: true # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_ENABLED
         # Redirect port for DNS
         port: 15053 # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_PORT
+      transparentProxyV2: false # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_TRANSPARENT_PROXY_V2
       # EBPF defines configuration for the ebpf, when transparent proxy is marked to be
       # installed using ebpf instead of iptables
       ebpf:
diff --git a/pkg/config/plugins/runtime/k8s/config.go b/pkg/config/plugins/runtime/k8s/config.go
index 884a28b57..9baa9e928 100644
--- a/pkg/config/plugins/runtime/k8s/config.go
+++ b/pkg/config/plugins/runtime/k8s/config.go
@@ -76,6 +76,7 @@ func DefaultKubernetesRuntimeConfig() *KubernetesRuntimeConfig {
 				Enabled: true,
 				Port:    15053,
 			},
+			TransparentProxyV2: false,
 			EBPF: EBPF{
 				Enabled:              false,
 				InstanceIPEnvVarName: "INSTANCE_IP",
@@ -145,6 +146,8 @@ type Injector struct {
 	// CaCertFile is CA certificate which will be used to verify a connection to the control plane
 	CaCertFile string     `yaml:"caCertFile" envconfig:"kuma_runtime_kubernetes_injector_ca_cert_file"`
 	BuiltinDNS BuiltinDNS `yaml:"builtinDNS"`
+	// TransparentProxyV2 enables the new transparent proxy engine for all workloads
+	TransparentProxyV2 bool `yaml:"transparentProxyV2" envconfig:"kuma_runtime_kubernetes_injector_transparent_proxy_v2"`
 	// EBPF is a configuration for ebpf if transparent proxy should be installed
 	// using ebpf instead of iptables
 	EBPF EBPF `yaml:"ebpf"`
diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/injector.go b/pkg/plugins/runtime/k8s/webhooks/injector/injector.go
index 3decc1825..74c501d62 100644
--- a/pkg/plugins/runtime/k8s/webhooks/injector/injector.go
+++ b/pkg/plugins/runtime/k8s/webhooks/injector/injector.go
@@ -401,7 +401,7 @@ func (i *KumaInjector) FindServiceAccountToken(podSpec *kube_core.PodSpec) *kube
 }
 
 func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container, error) {
-	podRedirect, err := tp_k8s.NewPodRedirectForPod(pod)
+	podRedirect, err := tp_k8s.NewPodRedirectForPod(i.cfg.TransparentProxyV2, pod)
 	if err != nil {
 		return kube_core.Container{}, err
 	}
diff --git a/pkg/transparentproxy/kubernetes/kubernetes.go b/pkg/transparentproxy/kubernetes/kubernetes.go
index 7e0d7b415..c61f342f9 100644
--- a/pkg/transparentproxy/kubernetes/kubernetes.go
+++ b/pkg/transparentproxy/kubernetes/kubernetes.go
@@ -45,7 +45,7 @@ type PodRedirect struct {
 	ExcludeOutboundUDPPortsForUIDs           []string
 }
 
-func NewPodRedirectForPod(pod *kube_core.Pod) (*PodRedirect, error) {
+func NewPodRedirectForPod(transparentProxyV2 bool, pod *kube_core.Pod) (*PodRedirect, error) {
 	var err error
 	podRedirect := &PodRedirect{}
 
@@ -99,7 +99,10 @@ func NewPodRedirectForPod(pod *kube_core.Pod) (*PodRedirect, error) {
 
 	podRedirect.UID, _ = metadata.Annotations(pod.Annotations).GetString(metadata.KumaSidecarUID)
 
-	podRedirect.ExperimentalTransparentProxyEngine, _, err = metadata.Annotations(pod.Annotations).GetEnabled(metadata.KumaTransparentProxyingExperimentalEngine)
+	podRedirect.ExperimentalTransparentProxyEngine, _, err = metadata.Annotations(pod.Annotations).GetEnabledWithDefault(
+		transparentProxyV2,
+		metadata.KumaTransparentProxyingExperimentalEngine,
+	)
 	if err != nil {
 		return nil, err
 	}


Test patch:
diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml
index 3670279bb..6cc108372 100644
--- a/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml
+++ b/app/kumactl/cmd/install/testdata/install-control-plane.dump-values.yaml
@@ -668,6 +668,8 @@ experimental:
   gatewayAPI: false
   # -- If true, it installs experimental new version of the CNI
   cni: false
+  # -- If true, use the new transparent proxy engine
+  transparentProxy: false
   # Configuration for the experimental ebpf mode for transparent proxy
   ebpf:
     # -- If true, ebpf will be used instead of using iptables to install/configure transparent proxy
diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go
index a8f968e48..bc2ac094f 100644
--- a/pkg/config/loader_test.go
+++ b/pkg/config/loader_test.go
@@ -182,6 +182,7 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.Runtime.Kubernetes.Injector.SidecarContainer.ReadinessProbe.InitialDelaySeconds).To(Equal(int32(41)))
 			Expect(cfg.Runtime.Kubernetes.Injector.BuiltinDNS.Enabled).To(Equal(true))
 			Expect(cfg.Runtime.Kubernetes.Injector.BuiltinDNS.Port).To(Equal(uint32(1053)))
+			Expect(cfg.Runtime.Kubernetes.Injector.TransparentProxyV2).To(Equal(true))
 			Expect(cfg.Runtime.Kubernetes.Injector.EBPF.Enabled).To(Equal(true))
 			Expect(cfg.Runtime.Kubernetes.Injector.EBPF.InstanceIPEnvVarName).To(Equal("FOO"))
 			Expect(cfg.Runtime.Kubernetes.Injector.EBPF.BPFFSPath).To(Equal("/run/kuma/bar"))
@@ -400,6 +401,7 @@ runtime:
       builtinDNS:
         enabled: true
         port: 1053
+      transparentProxyV2: true
       ebpf:
         enabled: true
         instanceIPEnvVarName: FOO
@@ -590,6 +592,7 @@ proxy:
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_READINESS_PROBE_INITIAL_DELAY_SECONDS": "41",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_ENABLED":                                     "true",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_PORT":                                        "1053",
+				"KUMA_RUNTIME_KUBERNETES_INJECTOR_TRANSPARENT_PROXY_V2":                                    "true",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_ENABLED":                                            "true",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_INSTANCE_IP_ENV_VAR_NAME":                           "FOO",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_EBPF_BPFFS_PATH":                                         "/run/kuma/bar",
diff --git a/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml b/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml
index 3edf54d36..d676f9399 100644
--- a/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml
+++ b/pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml
@@ -47,6 +47,7 @@ injector:
   builtinDNS:
     enabled: true
     port: 15053
+  transparentProxyV2: false
   ebpf:
     enabled: false
     instanceIPEnvVarName: INSTANCE_IP
diff --git a/pkg/transparentproxy/kubernetes/kubernetes_test.go b/pkg/transparentproxy/kubernetes/kubernetes_test.go
index 6db0e9fb4..c9c9afaa3 100644
--- a/pkg/transparentproxy/kubernetes/kubernetes_test.go
+++ b/pkg/transparentproxy/kubernetes/kubernetes_test.go
@@ -28,12 +28,13 @@ import (
 
 var _ = Describe("kubernetes", func() {
 	type testCaseKumactl struct {
-		pod         *kube_core.Pod
-		commandLine []string
+		transparentProxyV2 bool
+		pod                *kube_core.Pod
+		commandLine        []string
 	}
 
 	DescribeTable("should generate kumactl command line", func(given testCaseKumactl) {
-		podRedirect, err := kubernetes.NewPodRedirectForPod(given.pod)
+		podRedirect, err := kubernetes.NewPodRedirectForPod(given.transparentProxyV2, given.pod)
 		Expect(err).ToNot(HaveOccurred())
 
 		commandLine := podRedirect.AsKumactlCommandLine()
@@ -155,6 +156,33 @@ var _ = Describe("kubernetes", func() {
 				"--experimental-transparent-proxy-engine",
 			},
 		}),
+		Entry("should generate experimental engine if enabled even without annotation", testCaseKumactl{
+			transparentProxyV2: true,
+			pod: &kube_core.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						metadata.KumaTrafficExcludeOutboundPorts:                "11000",
+						metadata.KumaTransparentProxyingOutboundPortAnnotation:  "25100",
+						metadata.KumaTrafficExcludeInboundPorts:                 "12000",
+						metadata.KumaTransparentProxyingInboundPortAnnotation:   "25204",
+						metadata.KumaTransparentProxyingInboundPortAnnotationV6: "25206",
+						metadata.KumaSidecarUID:                                 "12345",
+					},
+				},
+			},
+			commandLine: []string{
+				"--redirect-outbound-port", "25100",
+				"--redirect-inbound=" + "true",
+				"--redirect-inbound-port", "25204",
+				"--redirect-inbound-port-v6", "25206",
+				"--kuma-dp-uid", "12345",
+				"--exclude-inbound-ports", "12000",
+				"--exclude-outbound-ports", "11000",
+				"--verbose",
+				"--skip-resolv-conf",
+				"--experimental-transparent-proxy-engine",
+			},
+		}),
 		Entry("should generate for Gateway", testCaseKumactl{
 			pod: &kube_core.Pod{
 				ObjectMeta: metav1.ObjectMeta{