Repository: Azure/ARO-RP
Pull request #2723 (2023-02-16 13:08:41)

policy: deny pod with master toleration taints
Which issue this PR addresses:
https://issues.redhat.com/browse/ARO-2222

What this PR does / why we need it:
Prevents non-privileged namespaces from having pods that have masters toleration taints.
non-privileged namespaces can match (openshift-logging|openshift-operators)

Test plan for issue:
Rego unit test + Gator test

Is there any documentation that needs to be updated for this PR?
May need to update the design doc for guardrail


Patch:
diff --git a/pkg/operator/controllers/guardrails/policies/.gitignore b/pkg/operator/controllers/guardrails/policies/.gitignore
new file mode 100644
index 000000000..0729f911d
--- /dev/null
+++ b/pkg/operator/controllers/guardrails/policies/.gitignore
@@ -0,0 +1,1 @@
+gkconstraints-test/
diff --git a/pkg/operator/controllers/guardrails/policies/gkconstraints/aro-master-toleration-pod-deny.yaml b/pkg/operator/controllers/guardrails/policies/gkconstraints/aro-master-toleration-pod-deny.yaml
new file mode 100644
index 000000000..8e1dadc13
--- /dev/null
+++ b/pkg/operator/controllers/guardrails/policies/gkconstraints/aro-master-toleration-pod-deny.yaml
@@ -0,0 +1,10 @@
+apiVersion: constraints.gatekeeper.sh/v1beta1
+kind: ARODenyMasterTolerationTaints
+metadata:
+  name: aro-deny-master-toleration-pods-in-nonprivileged-namespace
+spec:
+  enforcementAction: {{.Enforcement}}
+  match:
+    kinds:
+      - apiGroups: [""]
+        kinds: ["Pod"]
diff --git a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/aro-deny-master-toleration-taints.tmpl b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/aro-deny-master-toleration-taints.tmpl
new file mode 100644
index 000000000..75bea9933
--- /dev/null
+++ b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/aro-deny-master-toleration-taints.tmpl
@@ -0,0 +1,17 @@
+apiVersion: templates.gatekeeper.sh/v1
+kind: ConstraintTemplate
+metadata:
+  name: arodenymastertolerationtaints
+  annotations:
+    description: >-
+      Prevents non-privileged namespaces from having pods that have masters toleration taints.
+      non-privileged namespaces can match (openshift-logging|openshift-operators)
+spec:
+  crd:
+    spec:
+      names:
+        kind: ARODenyMasterTolerationTaints
+  targets:
+  - target: admission.k8s.gatekeeper.sh
+    rego: |
+{{ file.Read "gktemplates-src/aro-deny-master-toleration-taints/src.rego" | strings.Indent 8 | strings.TrimSuffix "\n" }}
diff --git a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/src.rego b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/src.rego
new file mode 100644
index 000000000..c1fb67abb
--- /dev/null
+++ b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/src.rego
@@ -0,0 +1,85 @@
+package arodenymastertolerationtaints
+
+import future.keywords.in
+
+violation[{"msg": msg}] {
+    # Check if the input namespace is a non-privileged namespace
+    ns := input.review.object.metadata.namespace
+    non_priv_namespace(ns)
+
+    # Check if the input operation is CREATE or UPDATE
+    input.review.operation in ["CREATE", "UPDATE"]
+
+    # Check if pod object has master toleration taints
+    tolerations := input.review.object.spec.tolerations
+    some toleration in tolerations
+    toleration.key in ["node-role.kubernetes.io/master", "node-role.kubernetes.io/control-plane"]
+
+    msg := "Create or update resources to have master toleration taints is not allowed in non-privileged namespaces"
+}
+
+non_priv_namespace(ns) {
+    not privileged_ns[ns]
+}
+
+privileged_ns = {
+    # ARO specific namespaces
+    "openshift-azure-logging",
+    "openshift-azure-operator",
+    "openshift-managed-upgrade-operator",
+    # OCP namespaces
+    "openshift",
+    "openshift-apiserver",
+    "openshift-apiserver-operator",
+    "openshift-authentication-operator",
+    "openshift-cloud-controller-manager",
+    "openshift-cloud-controller-manager-operator",
+    "openshift-cloud-credential-operator",
+    "openshift-cluster-csi-drivers",
+    "openshift-cluster-machine-approver",
+    "openshift-cluster-node-tuning-operator",
+    "openshift-cluster-samples-operator",
+    "openshift-cluster-storage-operator",
+    "openshift-cluster-version",
+    "openshift-config",
+    "openshift-config-managed",
+    "openshift-config-operator",
+    "openshift-console",
+    "openshift-console-operator",
+    "openshift-console-user-settings",
+    "openshift-controller-manager",
+    "openshift-controller-manager-operator",
+    "openshift-dns",
+    "openshift-dns-operator",
+    "openshift-etcd",
+    "openshift-etcd-operator",
+    "openshift-host-network",
+    "openshift-image-registry",
+    "openshift-ingress",
+    "openshift-ingress-canary",
+    "openshift-ingress-operator",
+    "openshift-insights",
+    "openshift-kni-infra",
+    "openshift-kube-apiserver",
+    "openshift-kube-apiserver-operator",
+    "openshift-kube-controller-manager",
+    "openshift-kube-controller-manager-operator",
+    "openshift-kube-scheduler",
+    "openshift-kube-scheduler-operator",
+    "openshift-kube-storage-version-migrator",
+    "openshift-kube-storage-version-migrator-operator",
+    "openshift-machine-api",
+    "openshift-machine-config-operator",
+    "openshift-marketplace",
+    "openshift-monitoring",
+    "openshift-multus",
+    "openshift-network-diagnostics",
+    "openshift-network-operator",
+    "openshift-oauth-apiserver",
+    "openshift-openstack-infra",
+    "openshift-operator-lifecycle-manager",
+    "openshift-ovirt-infra",
+    "openshift-sdn",
+    "openshift-service-ca",
+    "openshift-service-ca-operator",
+}
diff --git a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/suite.yaml b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/suite.yaml
new file mode 100644
index 000000000..f45f3adb1
--- /dev/null
+++ b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/suite.yaml
@@ -0,0 +1,13 @@
+kind: Suite
+apiVersion: test.gatekeeper.sh/v1alpha1
+metadata:
+  name: deny-master-toleration-taint
+tests:
+- name: deny-master-toleration-taint-pods-in-nonprivileged-namespaces
+  template: ../../gktemplates/aro-deny-master-toleration-taints.yaml
+  constraint: ../../gkconstraints-test/aro-master-toleration-pod-deny.yaml
+  cases:
+  - name: not-allowed-in-nonprivileged-namespaces
+    object: gator-test/not_allowed_pod_in_non_privledged_ns.yaml
+    assertions:
+    - violations: yes
diff --git a/pkg/operator/controllers/guardrails/policies/gktemplates/aro-deny-master-toleration-taints.yaml b/pkg/operator/controllers/guardrails/policies/gktemplates/aro-deny-master-toleration-taints.yaml
new file mode 100644
index 000000000..664ef96e4
--- /dev/null
+++ b/pkg/operator/controllers/guardrails/policies/gktemplates/aro-deny-master-toleration-taints.yaml
@@ -0,0 +1,101 @@
+apiVersion: templates.gatekeeper.sh/v1
+kind: ConstraintTemplate
+metadata:
+  name: arodenymastertolerationtaints
+  annotations:
+    description: >-
+      Prevents non-privileged namespaces from having pods that have masters toleration taints.
+      non-privileged namespaces can match (openshift-logging|openshift-operators)
+spec:
+  crd:
+    spec:
+      names:
+        kind: ARODenyMasterTolerationTaints
+  targets:
+  - target: admission.k8s.gatekeeper.sh
+    rego: |
+        package arodenymastertolerationtaints
+
+        import future.keywords.in
+
+        violation[{"msg": msg}] {
+            # Check if the input namespace is a non-privileged namespace
+            ns := input.review.object.metadata.namespace
+            non_priv_namespace(ns)
+
+            # Check if the input operation is CREATE or UPDATE
+            input.review.operation in ["CREATE", "UPDATE"]
+
+            # Check if pod object has master toleration taints
+            tolerations := input.review.object.spec.tolerations
+            some toleration in tolerations
+            toleration.key in ["node-role.kubernetes.io/master", "node-role.kubernetes.io/control-plane"]
+
+            msg := "Create or update resources to have master toleration taints is not allowed in non-privileged namespaces"
+        }
+
+        non_priv_namespace(ns) {
+            not privileged_ns[ns]
+        }
+
+        privileged_ns = {
+            # ARO specific namespaces
+            "openshift-azure-logging",
+            "openshift-azure-operator",
+            "openshift-managed-upgrade-operator",
+            # OCP namespaces
+            "openshift",
+            "openshift-apiserver",
+            "openshift-apiserver-operator",
+            "openshift-authentication-operator",
+            "openshift-cloud-controller-manager",
+            "openshift-cloud-controller-manager-operator",
+            "openshift-cloud-credential-operator",
+            "openshift-cluster-csi-drivers",
+            "openshift-cluster-machine-approver",
+            "openshift-cluster-node-tuning-operator",
+            "openshift-cluster-samples-operator",
+            "openshift-cluster-storage-operator",
+            "openshift-cluster-version",
+            "openshift-config",
+            "openshift-config-managed",
+            "openshift-config-operator",
+            "openshift-console",
+            "openshift-console-operator",
+            "openshift-console-user-settings",
+            "openshift-controller-manager",
+            "openshift-controller-manager-operator",
+            "openshift-dns",
+            "openshift-dns-operator",
+            "openshift-etcd",
+            "openshift-etcd-operator",
+            "openshift-host-network",
+            "openshift-image-registry",
+            "openshift-ingress",
+            "openshift-ingress-canary",
+            "openshift-ingress-operator",
+            "openshift-insights",
+            "openshift-kni-infra",
+            "openshift-kube-apiserver",
+            "openshift-kube-apiserver-operator",
+            "openshift-kube-controller-manager",
+            "openshift-kube-controller-manager-operator",
+            "openshift-kube-scheduler",
+            "openshift-kube-scheduler-operator",
+            "openshift-kube-storage-version-migrator",
+            "openshift-kube-storage-version-migrator-operator",
+            "openshift-machine-api",
+            "openshift-machine-config-operator",
+            "openshift-marketplace",
+            "openshift-monitoring",
+            "openshift-multus",
+            "openshift-network-diagnostics",
+            "openshift-network-operator",
+            "openshift-oauth-apiserver",
+            "openshift-openstack-infra",
+            "openshift-operator-lifecycle-manager",
+            "openshift-ovirt-infra",
+            "openshift-sdn",
+            "openshift-service-ca",
+            "openshift-service-ca-operator",
+        }


Test patch:
diff --git a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-labels/gator-test/no-label-machine-allowed.yaml b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-labels/gator-test/no-label-machine-allowed.yaml
index 520795d9a..ffc30d862 100644
--- a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-labels/gator-test/no-label-machine-allowed.yaml
+++ b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-labels/gator-test/no-label-machine-allowed.yaml
@@ -53,4 +53,3 @@ request:
     kind: CreateOptions
     apiVersion: meta.k8s.io/v1
   object:
-
diff --git a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/gator-test/not_allowed_pod_in_non_privledged_ns.yaml b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/gator-test/not_allowed_pod_in_non_privledged_ns.yaml
new file mode 100644
index 000000000..ed1eef704
--- /dev/null
+++ b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/gator-test/not_allowed_pod_in_non_privledged_ns.yaml
@@ -0,0 +1,46 @@
+kind: AdmissionReview
+apiVersion: admission.k8s.io/v1
+request:
+  uid: 46cf42b9-c413-4e04-ab23-777dac840bf5
+  kind:
+    group: ''
+    version: v1
+    kind: Pod
+  resource:
+    group: ''
+    version: v1
+    resource: pods
+  requestKind:
+    group: ''
+    version: v1
+    kind: Pod
+  requestResource:
+    group: ''
+    version: v1
+    resource: pods
+  name: demo-pod
+  operation: CREATE
+  userInfo:
+    username: kube-review
+    uid: eede9e7b-1854-4635-b5c0-029598ec6865
+  object:
+    kind: Pod
+    apiVersion: v1
+    metadata:
+      name: demo-pod
+      namespace: customer-namespace
+      creationTimestamp:
+    spec:
+      containers:
+      - name: demo-container
+        image: nginx
+        resources: {}
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    status: {}
+  oldObject:
+  options:
+    kind: CreateOptions
+    apiVersion: meta.k8s.io/v1
diff --git a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/src_test.rego b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/src_test.rego
new file mode 100644
index 000000000..f728fa292
--- /dev/null
+++ b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-master-toleration-taints/src_test.rego
@@ -0,0 +1,70 @@
+package arodenymastertolerationtaints
+
+
+test_input_allowed_in_privileged_ns_with_master_taint {
+    input := { 
+        "review": fake_input_review("openshift-config", "CREATE", "node-role.kubernetes.io/worker", "node-role.kubernetes.io/master")
+    }
+    results := violation with input as input
+    count(results) == 0
+}
+
+test_input_allowed_in_nonprivileged_ns_with_no_master_taint {
+    input := { 
+        "review": fake_input_review("customer", "CREATE", "node-role.kubernetes.io/worker", "node-role.kubernetes.io/worker")
+    }
+    results := violation with input as input
+    count(results) == 0
+}
+
+test_input_allowed_in_nonprivileged_ns_with_delete_operation {
+    input := { 
+        "review": fake_input_review("customer", "DELETE", "node-role.kubernetes.io/worker", "node-role.kubernetes.io/control-plane")
+    }
+    results := violation with input as input
+    count(results) == 0
+}
+
+test_input_not_allowed_in_nonprivileged_ns_with_create_operation {
+    input := { 
+        "review": fake_input_review("customer", "CREATE", "node-role.kubernetes.io/worker", "node-role.kubernetes.io/master")
+    }
+    results := violation with input as input
+    count(results) == 1
+}
+
+test_input_not_allowed_in_nonprivileged_ns_with_update_operation {
+    input := { 
+        "review": fake_input_review("customer", "UPDATE", "node-role.kubernetes.io/worker", "node-role.kubernetes.io/master")
+    }
+    results := violation with input as input
+    count(results) == 1
+}
+
+
+fake_input_review(namespace, operation, taint_key_one, taint_key_two) = review {
+    review = {
+        "operation": operation,
+        "kind": {
+            "kind": "Pod"
+        },
+        "object": {
+            "metadata": {
+                "namespace": namespace
+            },
+            "spec": {
+                "tolerations": [
+                    {
+                        "key": taint_key_one,
+                        "effect": "NoExecute"
+                    },
+                    {
+                        "key": taint_key_two,
+                        "effect": "NoExecute"
+                    }
+                ]
+            }
+        }
+        
+    }
+}
diff --git a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-privileged-namespace/gator-test/ns_disallowed_deploy.yaml b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-privileged-namespace/gator-test/ns_disallowed_deploy.yaml
index a6158211d..89da78cd2 100644
--- a/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-privileged-namespace/gator-test/ns_disallowed_deploy.yaml
+++ b/pkg/operator/controllers/guardrails/policies/gktemplates-src/aro-deny-privileged-namespace/gator-test/ns_disallowed_deploy.yaml
@@ -9,8 +9,8 @@ spec:
   selector:
     matchLabels:
       app: nginx
-  replicas: 3 
-  template: 
+  replicas: 3
+  template:
     metadata:
       labels:
         app: nginx