Repository: vmware-tanzu/tanzu-framework (language: go)

Issue:
Customize deployment of tkr-service package
(This is used to request new product features)

**Describe the feature request**
Make `hostNetwork`, `tolerations` and `nodeSelector` attributes of the deployment configurable in featuregates package. Apply the configurations when these values are passed at the package installation time.
Also 
- provide configuration option to configure the ports of the webhooks that are part of tkr-service package
- disable the metrics server on the webhooks that are part of the tkr-service package

**Describe alternatives you've considered**

**Affected product area (please put an X in all that apply)**

- [ ] APIs
- [x] Addons
- [ ] CLI
- [ ] Docs
- [ ] IAM
- [ ] Installation
- [ ] Plugin
- [ ] Security
- [ ] Test and Release
- [ ] User Experience

**Additional context**



Discussion / hints:

Fix patch:
diff --git a/packages/management/framework/bundle/config/pacakgeinstalls/addons-manager-pi.yaml b/packages/management/framework/bundle/config/packageinstalls/addons-manager-pi.yaml
similarity index 100%
rename from packages/management/framework/bundle/config/pacakgeinstalls/addons-manager-pi.yaml
rename to packages/management/framework/bundle/config/packageinstalls/addons-manager-pi.yaml
diff --git a/packages/management/framework/bundle/config/pacakgeinstalls/addons-manager-service-account.yaml b/packages/management/framework/bundle/config/packageinstalls/addons-manager-service-account.yaml
similarity index 100%
rename from packages/management/framework/bundle/config/pacakgeinstalls/addons-manager-service-account.yaml
rename to packages/management/framework/bundle/config/packageinstalls/addons-manager-service-account.yaml
diff --git a/packages/management/framework/bundle/config/pacakgeinstalls/cliplugins-pi.yaml b/packages/management/framework/bundle/config/packageinstalls/cliplugins-pi.yaml
similarity index 100%
rename from packages/management/framework/bundle/config/pacakgeinstalls/cliplugins-pi.yaml
rename to packages/management/framework/bundle/config/packageinstalls/cliplugins-pi.yaml
diff --git a/packages/management/framework/bundle/config/pacakgeinstalls/cliplugins-service-account.yaml b/packages/management/framework/bundle/config/packageinstalls/cliplugins-service-account.yaml
similarity index 100%
rename from packages/management/framework/bundle/config/pacakgeinstalls/cliplugins-service-account.yaml
rename to packages/management/framework/bundle/config/packageinstalls/cliplugins-service-account.yaml
diff --git a/packages/management/framework/bundle/config/pacakgeinstalls/featuregates-pi.yaml b/packages/management/framework/bundle/config/packageinstalls/featuregates-pi.yaml
similarity index 100%
rename from packages/management/framework/bundle/config/pacakgeinstalls/featuregates-pi.yaml
rename to packages/management/framework/bundle/config/packageinstalls/featuregates-pi.yaml
diff --git a/packages/management/framework/bundle/config/pacakgeinstalls/featuregates-service-account.yaml b/packages/management/framework/bundle/config/packageinstalls/featuregates-service-account.yaml
similarity index 100%
rename from packages/management/framework/bundle/config/pacakgeinstalls/featuregates-service-account.yaml
rename to packages/management/framework/bundle/config/packageinstalls/featuregates-service-account.yaml
diff --git a/packages/management/framework/bundle/config/pacakgeinstalls/tkr-service-pi.yaml b/packages/management/framework/bundle/config/packageinstalls/tkr-service-pi.yaml
similarity index 100%
rename from packages/management/framework/bundle/config/pacakgeinstalls/tkr-service-pi.yaml
rename to packages/management/framework/bundle/config/packageinstalls/tkr-service-pi.yaml
diff --git a/packages/management/framework/bundle/config/pacakgeinstalls/tkr-service-service-account.yaml b/packages/management/framework/bundle/config/packageinstalls/tkr-service-service-account.yaml
similarity index 99%
rename from packages/management/framework/bundle/config/pacakgeinstalls/tkr-service-service-account.yaml
rename to packages/management/framework/bundle/config/packageinstalls/tkr-service-service-account.yaml
index 6f57e3805f..168af534ea 100644
--- a/packages/management/framework/bundle/config/pacakgeinstalls/tkr-service-service-account.yaml
+++ b/packages/management/framework/bundle/config/packageinstalls/tkr-service-service-account.yaml
@@ -137,6 +137,8 @@ rules:
       - create
       - update
       - get
+      - list
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
diff --git a/packages/management/framework/bundle/config/values.yaml b/packages/management/framework/bundle/config/values.yaml
index c915ae3f4a..e289613277 100644
--- a/packages/management/framework/bundle/config/values.yaml
+++ b/packages/management/framework/bundle/config/values.yaml
@@ -13,6 +13,13 @@ featureGatesPackageValues:
 tkrServicePackageValues:
   namespace:
   versionConstraints:
+  deployment:
+    hostNetwork: false
+    nodeSelector: null
+    tolerations: []
+    #! If hostNetwork is set to true the below two ports should be different
+    tkrConversionWebhookServerPort: 9443
+    tkrResolverWebhookServerPort: 9443
 addonsManagerPackageValues:
   versionConstraints:
   tanzuAddonsManager:
diff --git a/packages/management/tkg/bundle/config/values.yaml b/packages/management/tkg/bundle/config/values.yaml
index 44fe11afb6..e4bc94e038 100644
--- a/packages/management/tkg/bundle/config/values.yaml
+++ b/packages/management/tkg/bundle/config/values.yaml
@@ -24,6 +24,13 @@ frameworkPackage:
     namespace: tkg-system
     createNamespace: false
     versionConstraints:
+    deployment:
+      hostNetwork: false
+      nodeSelector: null
+      tolerations: []
+      #! If hostNetwork is set to true the below two ports should be different
+      tkrConversionWebhookServerPort: 9443
+      tkrResolverWebhookServerPort: 9443
   addonsManagerPackageValues:
     versionConstraints:
     tanzuAddonsManager:
@@ -34,7 +41,6 @@ frameworkPackage:
         priorityClassName: system-cluster-critical
         nodeSelector: null
         tolerations: []
-        healthzPort: 18316
       featureGates:
         clusterBootstrapController: false
   clipluginsPackageValues:
diff --git a/packages/management/tkr-service/bundle/config/upstream/service.yaml b/packages/management/tkr-service/bundle/config/upstream/service.yaml
index 7a0192b55d..8c0b4b8bab 100644
--- a/packages/management/tkr-service/bundle/config/upstream/service.yaml
+++ b/packages/management/tkr-service/bundle/config/upstream/service.yaml
@@ -8,11 +8,11 @@ metadata:
 spec:
   ports:
   - port: 443
-    targetPort: 9443
+    targetPort: webhook-server
   selector:
     app: tkr-resolver-cluster-webhook
 
-## TKR Conversion webhook
+#! TKR Conversion webhook
 ---
 apiVersion: v1
 kind: Service
@@ -22,6 +22,6 @@ metadata:
 spec:
   ports:
   - port: 443
-    targetPort: 9443
+    targetPort: webhook-server
   selector:
     app: tkr-conversion-webhook
diff --git a/packages/management/tkr-service/bundle/config/upstream/tkr-conversion-webhook-deployment.yaml b/packages/management/tkr-service/bundle/config/upstream/tkr-conversion-webhook-deployment.yaml
index 375e8ba866..bcd6a59879 100644
--- a/packages/management/tkr-service/bundle/config/upstream/tkr-conversion-webhook-deployment.yaml
+++ b/packages/management/tkr-service/bundle/config/upstream/tkr-conversion-webhook-deployment.yaml
@@ -26,6 +26,9 @@ spec:
         name: manager
         command:
         - /manager
+        args:
+        - --metrics-bind-addr=0
+        - #@ "--webhook-server-port={}".format(data.values.deployment.tkrConversionWebhookServerPort)
         resources:
           limits:
             cpu: 100m
@@ -34,7 +37,7 @@ spec:
             cpu: 100m
             memory: 20Mi
         ports:
-        - containerPort: 9443
+        - containerPort: #@ data.values.deployment.tkrConversionWebhookServerPort
           name: webhook-server
           protocol: TCP
         volumeMounts:
@@ -46,3 +49,15 @@ spec:
       - name: cert
         secret:
           secretName: tkr-conversion-webhook-service-cert
+      #@ if hasattr(data.values, 'deployment') and hasattr(data.values.deployment, 'hostNetwork') and data.values.deployment.hostNetwork:
+      #@overlay/match missing_ok=True
+      hostNetwork: true
+      #@ end
+      #@ if hasattr(data.values, 'deployment') and hasattr(data.values.deployment, 'tolerations') and data.values.deployment.tolerations:
+      #@overlay/match missing_ok=True
+      tolerations: #@ data.values.deployment.tolerations
+      #@ end
+      #@ if hasattr(data.values, 'deployment') and hasattr(data.values.deployment, 'nodeSelector') and data.values.deployment.nodeSelector:
+      #@overlay/match missing_ok=True
+      nodeSelector: #@ data.values.deployment.nodeSelector
+      #@ end
diff --git a/packages/management/tkr-service/bundle/config/upstream/tkr-resolver-cluster-webhook-deployment.yaml b/packages/management/tkr-service/bundle/config/upstream/tkr-resolver-cluster-webhook-deployment.yaml
index 157d9f938d..fd35997789 100644
--- a/packages/management/tkr-service/bundle/config/upstream/tkr-resolver-cluster-webhook-deployment.yaml
+++ b/packages/management/tkr-service/bundle/config/upstream/tkr-resolver-cluster-webhook-deployment.yaml
@@ -28,6 +28,9 @@ spec:
         name: manager
         command:
         - /manager
+        args:
+        - --metrics-bind-addr=0
+        - #@ "--webhook-server-port={}".format(data.values.deployment.tkrResolverWebhookServerPort)
         resources:
           limits:
             cpu: 100m
@@ -36,7 +39,7 @@ spec:
             cpu: 100m
             memory: 20Mi
         ports:
-        - containerPort: 9443
+        - containerPort: #@ data.values.deployment.tkrResolverWebhookServerPort
           name: webhook-server
           protocol: TCP
         volumeMounts:
@@ -49,3 +52,15 @@ spec:
       - name: cert
         secret:
           secretName: tkr-resolver-cluster-webhook-service-cert
+      #@ if hasattr(data.values, 'deployment') and hasattr(data.values.deployment, 'hostNetwork') and data.values.deployment.hostNetwork:
+      #@overlay/match missing_ok=True
+      hostNetwork: true
+      #@ end
+      #@ if hasattr(data.values, 'deployment') and hasattr(data.values.deployment, 'tolerations') and data.values.deployment.tolerations:
+      #@overlay/match missing_ok=True
+      tolerations: #@ data.values.deployment.tolerations
+      #@ end
+      #@ if hasattr(data.values, 'deployment') and hasattr(data.values.deployment, 'nodeSelector') and data.values.deployment.nodeSelector:
+      #@overlay/match missing_ok=True
+      nodeSelector: #@ data.values.deployment.nodeSelector
+      #@ end
diff --git a/packages/management/tkr-service/bundle/config/values.yaml b/packages/management/tkr-service/bundle/config/values.yaml
index 365d28bb1e..f1ac8edeac 100644
--- a/packages/management/tkr-service/bundle/config/values.yaml
+++ b/packages/management/tkr-service/bundle/config/values.yaml
@@ -1,4 +1,11 @@
 #@data/values
 #@overlay/match-child-defaults missing_ok=True
 ---
-namespace: tkr-system
+namespace: tkg-system
+deployment:
+  hostNetwork: false
+  nodeSelector: null
+  tolerations: []
+  #! If hostNetwork is set to true the below two ports should be different
+  tkrConversionWebhookServerPort: 9443
+  tkrResolverWebhookServerPort: 9443
diff --git a/pkg/v2/tkr/webhook/cluster/tkr-resolver/main.go b/pkg/v2/tkr/webhook/cluster/tkr-resolver/main.go
index 9eaf5ab497..2667a69740 100644
--- a/pkg/v2/tkr/webhook/cluster/tkr-resolver/main.go
+++ b/pkg/v2/tkr/webhook/cluster/tkr-resolver/main.go
@@ -36,7 +36,11 @@ func init() {
 
 func main() {
 	var webhookCertDir string
+	var metricsAddr string
+	var webhookServerPort int
 	flag.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs/", "Webhook cert directory.")
+	flag.StringVar(&metricsAddr, "metrics-bind-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.IntVar(&webhookServerPort, "webhook-server-port", 9443, "The port that the webhook server serves at.")
 
 	opts := zap.Options{
 		Development: true,
@@ -51,8 +55,10 @@ func main() {
 	// Setup a Manager
 	setupLog.Info("setting up manager")
 	mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{
-		Scheme:  scheme,
-		CertDir: webhookCertDir,
+		Scheme:             scheme,
+		MetricsBindAddress: metricsAddr,
+		CertDir:            webhookCertDir,
+		Port:               webhookServerPort,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to set up controller manager")
diff --git a/pkg/v2/tkr/webhook/tkr-conversion/main.go b/pkg/v2/tkr/webhook/tkr-conversion/main.go
index 6237ee949f..d50af1657b 100644
--- a/pkg/v2/tkr/webhook/tkr-conversion/main.go
+++ b/pkg/v2/tkr/webhook/tkr-conversion/main.go
@@ -34,7 +34,11 @@ func init() {
 
 func main() {
 	var webhookCertDir string
+	var metricsAddr string
+	var webhookServerPort int
 	flag.StringVar(&webhookCertDir, "webhook-cert-dir", "/tmp/k8s-webhook-server/serving-certs/", "Webhook cert directory.")
+	flag.StringVar(&metricsAddr, "metrics-bind-addr", ":8080", "The address the metric endpoint binds to.")
+	flag.IntVar(&webhookServerPort, "webhook-server-port", 9443, "The port that the webhook server serves at.")
 
 	opts := zap.Options{
 		Development: true,
@@ -49,8 +53,10 @@ func main() {
 	// Setup a Manager
 	setupLog.Info("setting up manager")
 	mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{
-		Scheme:  scheme,
-		CertDir: webhookCertDir,
+		Scheme:             scheme,
+		MetricsBindAddress: metricsAddr,
+		CertDir:            webhookCertDir,
+		Port:               webhookServerPort,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to set up controller manager")


Test patch: