Repository: kinvolk/lokomotive
Pull request #1540 (2021-07-26 11:51:37)

prometheus-operator config update
Add user-configurable tolerations for prometheus operator and it's
components.

Change configuration for alertmanager. Now it is configured in a
different block

closes: #1527 
Signed-off-by: knrt10 <kautilya@kinvolk.io>

### Release notes

Alert manager is not configured as a different block

```hcl
# old
alertmanager_retention    = "360h"
alertmanager_external_url = "https://api.example.com/alertmanager"
alertmanager_config       = file("alertmanager-config.yaml")
alertmanager_node_selector = {
  "kubernetes.io/hostname" = "worker3"
}

# new
alertmanager {
  retention    = "360h"
  external_url = "https://api.example.com/alertmanager"
  config       = file("alertmanager-config.yaml")
  node_selector = {
    "kubernetes.io/hostname" = "worker3"
  }
}
```

Operator is also now configured as a new block

```hcl
# old
prometheus_operator_node_selector = {
  "kubernetes.io/hostname" = "worker3"
}

# new
operator {
  node_selector = {
    "kubernetes.io/hostname" = "worker3"
  }
}
```

knrt10: @invidian do we want package tests or e2e test?
invidian: I suggest adding:
```patch
diff --git a/pkg/components/prometheus-operator/component_test.go b/pkg/components/prometheus-operator/component_test.go
index 93c2911a..c161b7d9 100644
--- a/pkg/components/prometheus-operator/component_test.go
+++ b/pkg/components/prometheus-operator/component_test.go
@@ -24,6 +24,8 @@ import (

 //nolint:funlen
 func TestRenderManifest(t *testing.T) {
+       t.Parallel()
+
        tests := []struct {
                desc    string
                hcl     string
@@ -143,6 +145,8 @@ component "prometheus-operator" {
        for _, tc := range tests {
                tc := tc
                t.Run(tc.desc, func(t *testing.T) {
+                       t.Parallel()
+
                        b, d := util.GetComponentBody(tc.hcl, Name)
                        if d != nil {
                                t.Fatalf("error getting component body: %v", d)
@@ -177,6 +181,8 @@ component "prometheus-operator" {

 //nolint:funlen
 func TestConversion(t *testing.T) {
+       t.Parallel()
+
        testCases := []struct {
                name                 string
                inputConfig          string
```

This cuts down test execution time from 14 seconds to 4 seconds on my machine. Still slow, but better.
invidian: @knrt10 changes looks good to me, but I think some other maintainer should approve it for merging.
knrt10: @invidian I think you said Looks OK and submitted it via request changes. So had to request again 😄 
surajssd: @knrt10 can you rebase on the current master the ARM issue will be solved.

Patch:
diff --git a/docs/configuration-reference/components/prometheus-operator.md b/docs/configuration-reference/components/prometheus-operator.md
index e406e13f..a6334be6 100644
--- a/docs/configuration-reference/components/prometheus-operator.md
+++ b/docs/configuration-reference/components/prometheus-operator.md
@@ -11,15 +11,15 @@ instances.
 
 ## Prerequisites
 
-* A Lokomotive cluster with a
+- A Lokomotive cluster with a
   [PersistentVolume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) plugin, e.g.
   [OpenEBS](openebs-operator.md) or one of the
   [built-in](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#types-of-persistent-volumes)
   plugins.
 
-* An ingress controller such as [Contour](contour.md) for HTTP ingress.
+- An ingress controller such as [Contour](contour.md) for HTTP ingress.
 
-* The [cert-manager component](cert-manager.md) is required for enabling HTTPS.
+- The [cert-manager component](cert-manager.md) is required for enabling HTTPS.
 
 ## Configuration
 
@@ -41,6 +41,21 @@ component "prometheus-operator" {
     }
   }
 
+  operator {
+    tolerations {
+      key      = "lokomotive.io/operator"
+      operator = "Equal"
+      value    = "test"
+      effect   = "NoSchedule"
+    }
+    admission_webhook_tolerations {
+      key      = "lokomotive.io/operator-admission-webhook"
+      operator = "Equal"
+      value    = "test"
+      effect   = "NoSchedule"
+    }
+  }
+
   prometheus {
     metrics_retention = "14d"
     storage_size      = "50Gi"
@@ -59,11 +74,19 @@ component "prometheus-operator" {
     }
   }
 
-  alertmanager_retention    = "360h"
-  alertmanager_external_url = "https://api.example.com/alertmanager"
-  alertmanager_config       = file("alertmanager-config.yaml")
-  alertmanager_node_selector = {
-    "kubernetes.io/hostname" = "worker3"
+  alertmanager {
+    retention    = "360h"
+    external_url = "https://api.example.com/alertmanager"
+    config       = file("alertmanager-config.yaml")
+    node_selector = {
+      "kubernetes.io/hostname" = "worker3"
+    }
+    tolerations {
+      key      = "lokomotive.io/alertmanager"
+      operator = "Equal"
+      value    = "test"
+      effect   = "NoSchedule"
+    }
   }
 }
 ```
@@ -75,22 +98,22 @@ information.
 **Note**: Make sure the whole file is indented two spaces. That is, there are two spaces before the top level block.
 
 ```yaml
-  config:
-    global:
-      resolve_timeout: 5m
-    route:
-      group_by:
+config:
+  global:
+    resolve_timeout: 5m
+  route:
+    group_by:
       - job
-      group_wait: 30s
-      group_interval: 5m
-      repeat_interval: 12h
-      receiver: 'null'
-      routes:
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 12h
+    receiver: "null"
+    routes:
       - match:
           alertname: Watchdog
-        receiver: 'null'
-    receivers:
-    - name: 'null'
+        receiver: "null"
+  receivers:
+    - name: "null"
 ```
 
 **NOTE**: Ensure the file `alertmanager_config.yaml` is added to `.gitignore` to avoid any accidental exposure
@@ -122,41 +145,44 @@ EOF
 
 Table of all the arguments accepted by the component.
 
-| Argument                                     | Description                                                                                                                                                                                                                                         |                                                                                                                       Default                                                                                                                       |    Type     | Required |
-|----------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-----------:|:--------:|
-| `namespace`                                  | Namespace to deploy the Prometheus Operator.                                                                                                                                                                                                        |                                                                                                                    `monitoring`                                                                                                                     |   string    |  false   |
-| `grafana.admin_password`                     | Password for `admin` user in Grafana. If not provided it is auto generated and stored in secret `prometheus-operator-grafana`.                                                                                                                      |                                                                                                                          -                                                                                                                          |   string    |  false   |
-| `grafana.secret_env`                         | Sensitive environment variables passed to Grafana pod and stored as secret. Read more on manipulating `grafana.ini` using env var [here](https://grafana.com/docs/grafana/latest/installation/configuration/#configure-with-environment-variables). |                                                                                                                          -                                                                                                                          | map(string) |  false   |
-| `grafana.ingress.host`                       | Ingress URL host to expose Grafana over the internet. **NOTE:** When running on Equinix Metal, a DNS entry pointing at the ingress controller needs to be created.                                                                                         |                                                                                                                          -                                                                                                                   |   string    |   true   |
-| `grafana.ingress.class`                      | Ingress class to use for Grafana ingress.                                                                                                                                                                                                           |                                                                                                                      `contour`                                                                                                                      |   string    |  false   |
-| `grafana.ingress.certmanager_cluster_issuer` | `ClusterIssuer` to be used by cert-manager while issuing TLS certificates. Supported values: `letsencrypt-production`, `letsencrypt-staging`.                                                                                                       |                                                                                                              `letsencrypt-production`                                                                                                               |   string    |  false   |
-| `prometheus_operator_node_selector`          | Node selector to specify nodes where the Prometheus Operator pods should be deployed.                                                                                                                                                               |                                                                                                                         {}                                                                                                                          | map(string) |  false   |
-| `prometheus.metrics_retention`               | Time duration Prometheus shall retain data for. Must match the regular expression `[0-9]+(ms\|s\|m\|h\|d\|w\|y)` (milliseconds, seconds, minutes, hours, days, weeks and years).                                                                    |                                                                                                                        `10d`                                                                                                                        |   string    |  false   |
-| `prometheus.node_selector`                   | Node selector to specify nodes where the Prometheus pods should be deployed.                                                                                                                                                                        |                                                                                                                         {}                                                                                                                          | map(string) |  false   |
-| `prometheus.storage_size`                    | Storage capacity for the Prometheus in bytes. You can express storage as a fixed-point integer using one of these suffixes: E, P, T, G, M, K. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki.                                |                                                                                                                       "50Gi"                                                                                                                        |   string    |  false   |
-| `prometheus.watch_labeled_service_monitors`  | By default prometheus operator watches only the ServiceMonitor objects in the cluster that are labeled `release: prometheus-operator`. If set to `false` then all the ServiceMonitors will be watched.                                              |                                                                                                                       `true`                                                                                                                        |    bool     |  false   |
-| `prometheus.watch_labeled_prometheus_rules`  | By default prometheus operator watches only the PrometheusRule objects in the cluster that are labeled `release: prometheus-operator` and `app: kube-prometheus-stack`. If set to `false` then all the PrometheusRule will be watched.              |                                                                                                                       `true`                                                                                                                        |    bool     |  false   |
-| `prometheus.external_labels`                 | This is the Prometheus parameter with the same name. The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).                                                            |                                                                                                                          -                                                                                                                          | map(string) |  false   |
-| `prometheus.ingress.host`                       | Ingress URL host to expose Prometheus over the internet. **NOTE:** When running on Equinix Metal, a DNS entry pointing at the ingress controller needs to be created.                                                                                   |                                                                                                                          -                                                                                                                   |   string    |   true   |
-| `prometheus.ingress.class`                      | Ingress class to use for Prometheus ingress.                                                                                                                                                                                                     |                                                                                                                      `contour`                                                                                                                      |   string    |  false   |
-| `prometheus.ingress.certmanager_cluster_issuer` | `ClusterIssuer` to be used by cert-manager while issuing TLS certificates. Supported values: `letsencrypt-production`, `letsencrypt-staging`.                                                                                                    |                                                                                                              `letsencrypt-production`                                                                                                               |   string    |  false   |
-| `prometheus.external_url`                    | The URL on which Prometheus will be accessible. If not provided, the URL is taken from `prometheus.ingress.host` with `https` as a scheme.                                                                                                          |                                                                                                                          -                                                                                                                          |   string    |  false   |
-| `alertmanager_retention`                     | Time duration Alertmanager shall retain data for. Must match the regular expression `[0-9]+(ms\|s\|m\|h)` (milliseconds, seconds, minutes and hours).                                                                                               |                                                                                                                       `120h`                                                                                                                        |   string    |  false   |
-| `alertmanager_external_url`                  | The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name.                                                           |                                                                                                                         ""                                                                                                                          |   string    |  false   |
-| `alertmanager_config`                        | Provide YAML file path to configure Alertmanager. See [https://prometheus.io/docs/alerting/configuration/#configuration-file](https://prometheus.io/docs/alerting/configuration/#configuration-file).                                               | `{"global":{"resolve_timeout":"5m"},"route":{"group_by":["job"],"group_wait":"30s","group_interval":"5m","repeat_interval":"12h","receiver":"null","routes":[{"match":{"alertname":"Watchdog"},"receiver":"null"}]},"receivers":[{"name":"null"}]}` |   string    |  false   |
-| `alertmanager_node_selector`                 | Node selector to specify nodes where the AlertManager pods should be deployed.                                                                                                                                                                      |                                                                                                                         {}                                                                                                                          | map(string) |  false   |
-| `alertmanager_storage_size`                  | Storage capacity for the Alertmanager in bytes. You can express storage as a fixed-point integer using one of these suffixes: E, P, T, G, M, K. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki.                              |                                                                                                                       "50Gi"                                                                                                                        |   string    |  false   |
-| `disable_webhooks`                           | Disables validation and mutation webhooks. This might be required on older versions of Kubernetes to install successfully.                                                                                                                          |                                                                                                                        false                                                                                                                        |    bool     |  false   |
-| `monitor`                                    | Block, which allows to disable scraping of individual Kubernetes components.                                                                                                                                                                        |                                                                                                                          -                                                                                                                          |   object    |  false   |
-| `monitor.etcd`                               | Controls if the default Prometheus instance should scrape etcd metrics.                                                                                                                                                                             |                                                                                                                        true                                                                                                                         |    bool     |  false   |
-| `monitor.kube_controller_manager`            | Controls if the default Prometheus instance should scrape kube-controller-manager metrics.                                                                                                                                                          |                                                                                                                        true                                                                                                                         |    bool     |  false   |
-| `monitor.kube_scheduler`                     | Controls if the default Prometheus instance should scrape kube-scheduler metrics.                                                                                                                                                                   |                                                                                                                        true                                                                                                                         |    bool     |  false   |
-| `monitor.kube_proxy`                         | Controls if the default Prometheus instance should scrape kube-proxy metrics.                                                                                                                                                                       |                                                                                                                        true                                                                                                                         |    bool     |  false   |
-| `monitor.kubelet`                            | Controls if the default Prometheus instance should scrape kubelet metrics.                                                                                                                                                                          |                                                                                                                        true                                                                                                                         |    bool     |  false   |
-| `coredns`                                    | Block, which allows to customize, how CoreDNS is scraped.                                                                                                                                                                                           |                                                                                                                          -                                                                                                                          |   object    |  false   |
-| `coredns.selector`                           | Defines, how CoreDNS pods should be selected for scraping.                                                                                                                                                                                          |                                                                                                    {"k8s-app":"coredns","tier":"control-plane"}                                                                                                     | map(string) |  false   |
-| `storage_class`                              | Storage Class to use for the storage allowed for Prometheus and Alertmanager.                                                                                                                                                                       |                                                                                                                          -                                                                                                                          |   string    |  false   |
-
+| Argument                                        | Description                                                                                                                                                                                                                                         |                                                                                                                       Default                                                                                                                       |                                                      Type                                                      | Required |
+|-------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------:|:--------:|
+| `namespace`                                     | Namespace to deploy the Prometheus Operator.                                                                                                                                                                                                        |                                                                                                                    `monitoring`                                                                                                                     |                                                     string                                                     |  false   |
+| `grafana.admin_password`                        | Password for `admin` user in Grafana. If not provided it is auto generated and stored in secret `prometheus-operator-grafana`.                                                                                                                      |                                                                                                                          -                                                                                                                          |                                                     string                                                     |  false   |
+| `grafana.secret_env`                            | Sensitive environment variables passed to Grafana pod and stored as secret. Read more on manipulating `grafana.ini` using env var [here](https://grafana.com/docs/grafana/latest/installation/configuration/#configure-with-environment-variables). |                                                                                                                          -                                                                                                                          |                                                  map(string)                                                   |  false   |
+| `grafana.ingress.host`                          | Ingress URL host to expose Grafana over the internet. **NOTE:** When running on Equinix Metal, a DNS entry pointing at the ingress controller needs to be created.                                                                                  |                                                                                                                          -                                                                                                                          |                                                     string                                                     |   true   |
+| `grafana.ingress.class`                         | Ingress class to use for Grafana ingress.                                                                                                                                                                                                           |                                                                                                                      `contour`                                                                                                                      |                                                     string                                                     |  false   |
+| `grafana.ingress.certmanager_cluster_issuer`    | `ClusterIssuer` to be used by cert-manager while issuing TLS certificates. Supported values: `letsencrypt-production`, `letsencrypt-staging`.                                                                                                       |                                                                                                              `letsencrypt-production`                                                                                                               |                                                     string                                                     |  false   |
+| `operator.node_selector`                        | Node selector to specify nodes where the Prometheus Operator pods should be deployed.                                                                                                                                                               |                                                                                                                         {}                                                                                                                          |                                                  map(string)                                                   |  false   |
+| `operator.tolerations`                          | Toleration that prometheus operator will tolerate.                                                                                                                                                                                                  |                                                                                                                          -                                                                                                                          | list(object({key = string, effect = string, operator = string, value = string, toleration_seconds = string })) |  false   |
+| `operator.admission_webhook_tolerations`        | Toleration that prometheus operator admission webhook patch job will tolerate.                                                                                                                                                                      |                                                                                                                          -                                                                                                                          | list(object({key = string, effect = string, operator = string, value = string, toleration_seconds = string })) |  false   |
+| `prometheus.metrics_retention`                  | Time duration Prometheus shall retain data for. Must match the regular expression `[0-9]+(ms\|s\|m\|h\|d\|w\|y)` (milliseconds, seconds, minutes, hours, days, weeks and years).                                                                    |                                                                                                                        `10d`                                                                                                                        |                                                     string                                                     |  false   |
+| `prometheus.node_selector`                      | Node selector to specify nodes where the Prometheus pods should be deployed.                                                                                                                                                                        |                                                                                                                         {}                                                                                                                          |                                                  map(string)                                                   |  false   |
+| `prometheus.tolerations`                        | Toleration that prometheus pods will tolerate.                                                                                                                                                                                                      |                                                                                                                          -                                                                                                                          | list(object({key = string, effect = string, operator = string, value = string, toleration_seconds = string })) |  false   |
+| `prometheus.storage_size`                       | Storage capacity for the Prometheus in bytes. You can express storage as a fixed-point integer using one of these suffixes: E, P, T, G, M, K. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki.                                |                                                                                                                       "50Gi"                                                                                                                        |                                                     string                                                     |  false   |
+| `prometheus.watch_labeled_service_monitors`     | By default prometheus operator watches only the ServiceMonitor objects in the cluster that are labeled `release: prometheus-operator`. If set to `false` then all the ServiceMonitors will be watched.                                              |                                                                                                                       `true`                                                                                                                        |                                                      bool                                                      |  false   |
+| `prometheus.watch_labeled_prometheus_rules`     | By default prometheus operator watches only the PrometheusRule objects in the cluster that are labeled `release: prometheus-operator` and `app: kube-prometheus-stack`. If set to `false` then all the PrometheusRule will be watched.              |                                                                                                                       `true`                                                                                                                        |                                                      bool                                                      |  false   |
+| `prometheus.external_labels`                    | This is the Prometheus parameter with the same name. The labels to add to any time series or alerts when communicating with external systems (federation, remote storage, Alertmanager).                                                            |                                                                                                                          -                                                                                                                          |                                                  map(string)                                                   |  false   |
+| `prometheus.ingress.host`                       | Ingress URL host to expose Prometheus over the internet. **NOTE:** When running on Equinix Metal, a DNS entry pointing at the ingress controller needs to be created.                                                                               |                                                                                                                          -                                                                                                                          |                                                     string                                                     |   true   |
+| `prometheus.ingress.class`                      | Ingress class to use for Prometheus ingress.                                                                                                                                                                                                        |                                                                                                                      `contour`                                                                                                                      |                                                     string                                                     |  false   |
+| `prometheus.ingress.certmanager_cluster_issuer` | `ClusterIssuer` to be used by cert-manager while issuing TLS certificates. Supported values: `letsencrypt-production`, `letsencrypt-staging`.                                                                                                       |                                                                                                              `letsencrypt-production`                                                                                                               |                                                     string                                                     |  false   |
+| `prometheus.external_url`                       | The URL on which Prometheus will be accessible. If not provided, the URL is taken from `prometheus.ingress.host` with `https` as a scheme.                                                                                                          |                                                                                                                          -                                                                                                                          |                                                     string                                                     |  false   |
+| `alertmanager.retention`                        | Time duration Alertmanager shall retain data for. Must match the regular expression `[0-9]+(ms\|s\|m\|h)` (milliseconds, seconds, minutes and hours).                                                                                               |                                                                                                                       `120h`                                                                                                                        |                                                     string                                                     |  false   |
+| `alertmanager.external_url`                     | The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name.                                                           |                                                                                                                         ""                                                                                                                          |                                                     string                                                     |  false   |
+| `alertmanager.config`                           | Provide YAML file path to configure Alertmanager. See [https://prometheus.io/docs/alerting/configuration/#configuration-file](https://prometheus.io/docs/alerting/configuration/#configuration-file).                                               | `{"global":{"resolve_timeout":"5m"},"route":{"group_by":["job"],"group_wait":"30s","group_interval":"5m","repeat_interval":"12h","receiver":"null","routes":[{"match":{"alertname":"Watchdog"},"receiver":"null"}]},"receivers":[{"name":"null"}]}` |                                                     string                                                     |  false   |
+| `alertmanager.node_selector`                    | Node selector to specify nodes where the AlertManager pods should be deployed.                                                                                                                                                                      |                                                                                                                         {}                                                                                                                          |                                                  map(string)                                                   |  false   |
+| `alertmanager.tolerations`                      | Toleration that AlertManager will tolerate.                                                                                                                                                                                                         |                                                                                                                          -                                                                                                                          | list(object({key = string, effect = string, operator = string, value = string, toleration_seconds = string })) |  false   |
+| `alertmanager.storage_size`                     | Storage capacity for the Alertmanager in bytes. You can express storage as a fixed-point integer using one of these suffixes: E, P, T, G, M, K. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki.                              |                                                                                                                       "50Gi"                                                                                                                        |                                                     string                                                     |  false   |
+| `disable_webhooks`                              | Disables validation and mutation webhooks. This might be required on older versions of Kubernetes to install successfully.                                                                                                                          |                                                                                                                        false                                                                                                                        |                                                      bool                                                      |  false   |
+| `monitor`                                       | Block, which allows to disable scraping of individual Kubernetes components.                                                                                                                                                                        |                                                                                                                          -                                                                                                                          |                                                     object                                                     |  false   |
+| `monitor.etcd`                                  | Controls if the default Prometheus instance should scrape etcd metrics.                                                                                                                                                                             |                                                                                                                        true                                                                                                                         |                                                      bool                                                      |  false   |
+| `monitor.kube_controller_manager`               | Controls if the default Prometheus instance should scrape kube-controller-manager metrics.                                                                                                                                                          |                                                                                                                        true                                                                                                                         |                                                      bool                                                      |  false   |
+| `monitor.kube_scheduler`                        | Controls if the default Prometheus instance should scrape kube-scheduler metrics.                                                                                                                                                                   |                                                                                                                        true                                                                                                                         |                                                      bool                                                      |  false   |
+| `monitor.kube_proxy`                            | Controls if the default Prometheus instance should scrape kube-proxy metrics.                                                                                                                                                                       |                                                                                                                        true                                                                                                                         |                                                      bool                                                      |  false   |
+| `monitor.kubelet`                               | Controls if the default Prometheus instance should scrape kubelet metrics.                                                                                                                                                                          |                                                                                                                        true                                                                                                                         |                                                      bool                                                      |  false   |
+| `coredns`                                       | Block, which allows to customize, how CoreDNS is scraped.                                                                                                                                                                                           |                                                                                                                          -                                                                                                                          |                                                     object                                                     |  false   |
+| `coredns.selector`                              | Defines, how CoreDNS pods should be selected for scraping.                                                                                                                                                                                          |                                                                                                    {"k8s-app":"coredns","tier":"control-plane"}                                                                                                     |                                                  map(string)                                                   |  false   |
+| `storage_class`                                 | Storage Class to use for the storage allowed for Prometheus and Alertmanager.                                                                                                                                                                       |                                                                                                                          -                                                                                                                          |                                                     string                                                     |  false   |
 
 ## Applying
 
@@ -181,11 +207,11 @@ metadata:
   namespace: openebs
 spec:
   endpoints:
-  - path: /metrics
-    port: exporter
+    - path: /metrics
+      port: exporter
   namespaceSelector:
     matchNames:
-    - openebs
+      - openebs
   selector:
     matchLabels:
       openebs.io/cas-type: cstor
diff --git a/pkg/components/prometheus-operator/component.go b/pkg/components/prometheus-operator/component.go
index 6f486bd9..d648eb24 100644
--- a/pkg/components/prometheus-operator/component.go
+++ b/pkg/components/prometheus-operator/component.go
@@ -55,6 +55,15 @@ type Grafana struct {
 	Ingress       *types.Ingress    `hcl:"ingress,block"`
 }
 
+// Operator object collects sub component Prometheus operator related information.
+type Operator struct {
+	AdmissionWebhookTolerations    []util.Toleration `hcl:"admission_webhook_tolerations,block"`
+	AdmissionWebhookTolerationsRaw string
+	NodeSelector                   map[string]string `hcl:"node_selector,optional"`
+	Tolerations                    []util.Toleration `hcl:"tolerations,block"`
+	TolerationsRaw                 string
+}
+
 // Prometheus object collects sub component Prometheus related information.
 type Prometheus struct {
 	MetricsRetention            string            `hcl:"metrics_retention,optional"`
@@ -65,6 +74,19 @@ type Prometheus struct {
 	Ingress                     *types.Ingress    `hcl:"ingress,block"`
 	ExternalLabels              map[string]string `hcl:"external_labels,optional"`
 	ExternalURL                 string            `hcl:"external_url,optional"`
+	Tolerations                 []util.Toleration `hcl:"tolerations,block"`
+	TolerationsRaw              string
+}
+
+// AlertManager object collects sub component AlertManager related information.
+type AlertManager struct {
+	Config         string            `hcl:"config,optional"`
+	ExternalURL    string            `hcl:"external_url,optional"`
+	NodeSelector   map[string]string `hcl:"node_selector,optional"`
+	Retention      string            `hcl:"retention,optional"`
+	StorageSize    string            `hcl:"storage_size,optional"`
+	Tolerations    []util.Toleration `hcl:"tolerations,block"`
+	TolerationsRaw string
 }
 
 type component struct {
@@ -72,15 +94,11 @@ type component struct {
 
 	Namespace string `hcl:"namespace,optional"`
 
-	PrometheusOperatorNodeSelector map[string]string `hcl:"prometheus_operator_node_selector,optional"`
+	Operator *Operator `hcl:"operator,block"`
 
 	Prometheus *Prometheus `hcl:"prometheus,block"`
 
-	AlertManagerRetention    string            `hcl:"alertmanager_retention,optional"`
-	AlertManagerExternalURL  string            `hcl:"alertmanager_external_url,optional"`
-	AlertManagerConfig       string            `hcl:"alertmanager_config,optional"`
-	AlertManagerNodeSelector map[string]string `hcl:"alertmanager_node_selector,optional"`
-	AlertManagerStorageSize  string            `hcl:"alertmanager_storage_size,optional"`
+	AlertManager *AlertManager `hcl:"alertmanager,block"`
 
 	StorageClass string `hcl:"storage_class,optional"`
 
@@ -120,10 +138,12 @@ func NewConfig() *component {
 			WatchLabeledServiceMonitors: true,
 			WatchLabeledPrometheusRules: true,
 		},
-		AlertManagerRetention:   "120h",
-		AlertManagerConfig:      defaultAlertManagerConfig,
-		AlertManagerStorageSize: "50Gi",
-		Namespace:               "monitoring",
+		AlertManager: &AlertManager{
+			Retention:   "120h",
+			Config:      defaultAlertManagerConfig,
+			StorageSize: "50Gi",
+		},
+		Namespace: "monitoring",
 		Monitor: &Monitor{
 			Etcd:                  true,
 			KubeControllerManager: true,
@@ -200,6 +220,28 @@ func (c *component) RenderManifests() (map[string]string, error) {
 		return nil, fmt.Errorf("retrieving chart from assets: %w", err)
 	}
 
+	c.Prometheus.TolerationsRaw, err = util.RenderTolerations(c.Prometheus.Tolerations)
+	if err != nil {
+		return nil, fmt.Errorf("rendering prometheus tolerations: %w", err)
+	}
+
+	if c.Operator != nil {
+		c.Operator.TolerationsRaw, err = util.RenderTolerations(c.Operator.Tolerations)
+		if err != nil {
+			return nil, fmt.Errorf("rendering operator tolerations: %w", err)
+		}
+
+		c.Operator.AdmissionWebhookTolerationsRaw, err = util.RenderTolerations(c.Operator.AdmissionWebhookTolerations) //nolint:lll
+		if err != nil {
+			return nil, fmt.Errorf("rendering operator admission webhook tolerations: %w", err)
+		}
+	}
+
+	c.AlertManager.TolerationsRaw, err = util.RenderTolerations(c.AlertManager.Tolerations)
+	if err != nil {
+		return nil, fmt.Errorf("rendering alertmanager tolerations: %w", err)
+	}
+
 	values, err := template.Render(chartValuesTmpl, c)
 	if err != nil {
 		return nil, fmt.Errorf("rendering chart values template: %w", err)
diff --git a/pkg/components/prometheus-operator/template.go b/pkg/components/prometheus-operator/template.go
index fb236692..687000e3 100644
--- a/pkg/components/prometheus-operator/template.go
+++ b/pkg/components/prometheus-operator/template.go
@@ -22,16 +22,19 @@ global:
       seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
 
 alertmanager:
-{{.AlertManagerConfig}}
+{{.AlertManager.Config}}
   alertmanagerSpec:
-    retention: {{.AlertManagerRetention}}
-    externalUrl: {{.AlertManagerExternalURL}}
-    {{ if .AlertManagerNodeSelector }}
+    retention: {{.AlertManager.Retention}}
+    externalUrl: {{.AlertManager.ExternalURL}}
+    {{ if .AlertManager.NodeSelector }}
     nodeSelector:
-      {{ range $key, $value := .AlertManagerNodeSelector }}
+      {{ range $key, $value := .AlertManager.NodeSelector }}
       {{ $key }}: {{ $value }}
       {{ end }}
     {{ end }}
+    {{ if .AlertManager.Tolerations }}
+    tolerations: {{ .AlertManager.TolerationsRaw }}
+    {{ end }}
     storage:
       volumeClaimTemplate:
         # This is done to reduce the name length of PVC that is autogenerated if metadata.Name is
@@ -45,7 +48,7 @@ alertmanager:
           accessModes: ["ReadWriteOnce"]
           resources:
             requests:
-              storage: "{{.AlertManagerStorageSize}}"
+              storage: "{{.AlertManager.StorageSize}}"
 
 grafana:
   plugins: "grafana-piechart-panel"
@@ -88,21 +91,24 @@ kubeEtcd:
   enabled: {{.Monitor.Etcd}}
 prometheus-node-exporter:
   service: {}
-{{ if (or .PrometheusOperatorNodeSelector .DisableWebhooks) }}
 prometheusOperator:
-  {{- if .DisableWebhooks }}
-  tlsProxy:
-    enabled: false
   admissionWebhooks:
-    enabled: false
-  {{- end }}
-  {{- if .PrometheusOperatorNodeSelector }}
+    enabled: {{ not .DisableWebhooks  }}
+  {{- with .Operator }}
+    {{- if .AdmissionWebhookTolerations }}
+    patch:
+      tolerations: {{ .AdmissionWebhookTolerationsRaw }}
+    {{- end }}
+  {{- if .NodeSelector }}
   nodeSelector:
-    {{ range $key, $value := .PrometheusOperatorNodeSelector }}
+    {{ range $key, $value := .NodeSelector }}
     {{ $key }}: {{ $value }}
     {{ end }}
   {{- end }}
-{{ end }}
+  {{- if .Tolerations }}
+  tolerations: {{ .TolerationsRaw }}
+  {{- end }}
+  {{- end }}
 prometheus:
   {{ if .Prometheus.Ingress }}
   ingress:
@@ -131,6 +137,9 @@ prometheus:
       {{ $key }}: {{ $value }}
       {{ end }}
     {{ end }}
+    {{ if .Prometheus.Tolerations }}
+    tolerations: {{ .Prometheus.TolerationsRaw }}
+    {{ end }}
     {{ if .Prometheus.ExternalLabels }}
     externalLabels:
       {{ range $key, $value := .Prometheus.ExternalLabels}}


Test patch:
diff --git a/pkg/components/prometheus-operator/component_test.go b/pkg/components/prometheus-operator/component_test.go
index c53e29cf..38d64da6 100644
--- a/pkg/components/prometheus-operator/component_test.go
+++ b/pkg/components/prometheus-operator/component_test.go
@@ -24,6 +24,8 @@ import (
 
 //nolint:funlen
 func TestRenderManifest(t *testing.T) {
+	t.Parallel()
+
 	tests := []struct {
 		desc    string
 		hcl     string
@@ -89,6 +91,52 @@ component "prometheus-operator" {
     }
   }
 }
+`,
+			wantErr: false,
+		},
+		{
+			desc: "prometheus operator and admission webhook toleration are set correctly",
+			hcl: `
+component "prometheus-operator" {
+  operator {
+    admission_webhook_tolerations {
+      key      = "lokomotive.io/operator-admission-webhook"
+      operator = "Equal"
+      value    = "test"
+      effect   = "NoSchedule"
+    }
+    tolerations {
+      key      = "lokomotive.io/operator"
+      operator = "Equal"
+      value    = "test"
+      effect   = "NoSchedule"
+    }
+  }
+}
+`,
+			wantErr: false,
+		},
+		{
+			desc: "prometheus and alertmanager toleration are set correctly",
+			hcl: `
+component "prometheus-operator" {
+  alertmanager {
+    tolerations {
+      key      = "lokomotive.io/alertmanager"
+      operator = "Equal"
+      value    = "test"
+      effect   = "NoSchedule"
+    }
+  }
+  prometheus{
+    tolerations {
+      key      = "lokomotive.io/prometheus"
+      operator = "Equal"
+      value    = "test"
+      effect   = "NoSchedule"
+    }
+  }
+}
 `,
 			wantErr: false,
 		},
@@ -97,6 +145,7 @@ component "prometheus-operator" {
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.desc, func(t *testing.T) {
+			t.Parallel()
 			b, d := util.GetComponentBody(tc.hcl, Name)
 			if d != nil {
 				t.Fatalf("error getting component body: %v", d)
@@ -131,6 +180,8 @@ component "prometheus-operator" {
 
 //nolint:funlen
 func TestConversion(t *testing.T) {
+	t.Parallel()
+
 	testCases := []struct {
 		name                 string
 		inputConfig          string
@@ -210,6 +261,46 @@ providers:
     path: /tmp/dashboards`,
 			jsonPath: "{.data.provider\\.yaml}",
 		},
+		{
+			name: "prometheus operator tolerations",
+			inputConfig: `
+		component "prometheus-operator" {
+			operator {
+				tolerations {
+					key      = "lokomotive.io/operator"
+					operator = "Equal"
+					value    = "test"
+					effect   = "NoSchedule"
+				}
+			}
+		}
+		`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "apps/v1", Kind: "Deployment", Name: "prometheus-operator-kube-p-operator",
+			},
+			expected: "lokomotive.io/operator",
+			jsonPath: "{.spec.template.spec.tolerations[0].key}",
+		},
+		{
+			name: "prometheus operator alertmanager tolerations",
+			inputConfig: `
+		component "prometheus-operator" {
+			alertmanager {
+				tolerations {
+					key      = "lokomotive.io/alertmanager"
+					operator = "Equal"
+					value    = "test"
+					effect   = "NoSchedule"
+				}
+			}
+		}
+		`,
+			expectedManifestName: k8sutil.ObjectMetadata{
+				Version: "monitoring.coreos.com/v1", Kind: "Alertmanager", Name: "prometheus-operator-kube-p-alertmanager",
+			},
+			expected: "lokomotive.io/alertmanager",
+			jsonPath: "{.spec.tolerations[0].key}",
+		},
 	}
 
 	for _, tc := range testCases {