Observability Red Hat Advanced Cluster Management for Kubernetes 2.12 Observability

Commands:
create namespace open-cluster-management-observability
DOCKER_CONFIG_JSON=`oc extract secret/multiclusterhub-operator-pull-secret -n open-cluster-management --to=-`
DOCKER_CONFIG_JSON=`oc extract secret/pull-secret -n openshift-config --to=-`
create secret generic multiclusterhub-operator-pull-secret -n open-cluster-management-observability --from-literal=.dockerconfigjson="USDDOCKER_CONFIG_JSON" --type=kubernetes.io/dockerconfigjson
create -f thanos-object-storage.yaml -n open-cluster-management-observability
apiVersion: v1 kind: Secret metadata: name: thanos-object-storage namespace: open-cluster-management-observability type: Opaque stringData: thanos.yaml: | type: s3 config: bucket: YOUR_S3_BUCKET endpoint: YOUR_S3_ENDPOINT 1 insecure: true access_key: YOUR_ACCESS_KEY secret_key: YOUR_SECRET_KEY
apiVersion: v1 kind: Secret metadata: name: thanos-object-storage namespace: open-cluster-management-observability type: Opaque stringData: thanos.yaml: | type: GCS config: bucket: YOUR_GCS_BUCKET service_account: YOUR_SERVICE_ACCOUNT
apiVersion: v1 kind: Secret metadata: name: thanos-object-storage namespace: open-cluster-management-observability type: Opaque stringData: thanos.yaml: | type: AZURE config: storage_account: YOUR_STORAGE_ACCT storage_account_key: YOUR_STORAGE_KEY container: YOUR_CONTAINER endpoint: blob.core.windows.net 1 max_retries: 0
apiVersion: v1 kind: Secret metadata: name: thanos-object-storage namespace: open-cluster-management-observability type: Opaque stringData: thanos.yaml: | type: s3 config: bucket: YOUR_RH_DATA_FOUNDATION_BUCKET endpoint: YOUR_RH_DATA_FOUNDATION_ENDPOINT 1 insecure: false access_key: YOUR_RH_DATA_FOUNDATION_ACCESS_KEY secret_key: YOUR_RH_DATA_FOUNDATION_SECRET_KEY
apiVersion: v1 kind: Secret metadata: name: thanos-object-storage namespace: open-cluster-management-observability type: Opaque stringData: thanos.yaml: | type: s3 config: bucket: YOUR_ROKS_S3_BUCKET endpoint: YOUR_ROKS_S3_ENDPOINT 1 insecure: true access_key: YOUR_ROKS_ACCESS_KEY secret_key: YOUR_ROKS_SECRET_KEY
export POLICY_VERSION=USD(date +"%m-%d-%y") export TRUST_POLICY_VERSION=USD(date +"%m-%d-%y") export CLUSTER_NAME=<my-cluster> export S3_BUCKET=USDCLUSTER_NAME-acm-observability export REGION=us-east-2 export NAMESPACE=open-cluster-management-observability export SA=tbd export SCRATCH_DIR=/tmp/scratch export OIDC_PROVIDER=USD(oc get authentication.config.openshift.io cluster -o json | jq -r .spec.serviceAccountIssuer| sed -e "s/^https:\/\///") export AWS_ACCOUNT_ID=USD(aws sts get-caller-identity --query Account --output text) export AWS_PAGER="" rm -rf USDSCRATCH_DIR mkdir -p USDSCRATCH_DIR
aws s3 mb s3://USDS3_BUCKET
{ "Version": "USDPOLICY_VERSION", "Statement": [ { "Sid": "Statement", "Effect": "Allow", "Action": [ "s3:ListBucket", "s3:GetObject", "s3:DeleteObject", "s3:PutObject", "s3:PutObjectAcl", "s3:CreateBucket", "s3:DeleteBucket" ], "Resource": [ "arn:aws:s3:::USDS3_BUCKET/*", "arn:aws:s3:::USDS3_BUCKET" ] } ] }
S3_POLICY=USD(aws iam create-policy --policy-name USDCLUSTER_NAME-acm-obs --policy-document file://USDSCRATCH_DIR/s3-policy.json --query 'Policy.Arn' --output text) echo USDS3_POLICY
{ "Version": "USDTRUST_POLICY_VERSION", "Statement": [ { "Effect": "Allow", "Principal": { "Federated": "arn:aws:iam::USD{AWS_ACCOUNT_ID}:oidc-provider/USD{OIDC_PROVIDER}" }, "Action": "sts:AssumeRoleWithWebIdentity", "Condition": { "StringEquals": { "USD{OIDC_PROVIDER}:sub": [ "system:serviceaccount:USD{NAMESPACE}:observability-thanos-query", "system:serviceaccount:USD{NAMESPACE}:observability-thanos-store-shard", "system:serviceaccount:USD{NAMESPACE}:observability-thanos-compact" "system:serviceaccount:USD{NAMESPACE}:observability-thanos-rule", "system:serviceaccount:USD{NAMESPACE}:observability-thanos-receive", ] } } } ] }
S3_ROLE=USD(aws iam create-role --role-name "USDCLUSTER_NAME-acm-obs-s3" --assume-role-policy-document file://USDSCRATCH_DIR/TrustPolicy.json --query "Role.Arn" --output text) echo USDS3_ROLE
aws iam attach-role-policy --role-name "USDCLUSTER_NAME-acm-obs-s3" --policy-arn USDS3_POLICY
apiVersion: v1 kind: Secret metadata: name: thanos-object-storage namespace: open-cluster-management-observability type: Opaque stringData: thanos.yaml: | type: s3 config: bucket: USDS3_BUCKET endpoint: s3.USDREGION.amazonaws.com signature_version2: false
YOUR_CLOUD_PROVIDER_ACCESS_KEY=USD(oc -n open-cluster-management-observability get secret <object-storage-secret> -o jsonpath="{.data.thanos\.yaml}" | base64 --decode | grep access_key | awk '{print USD2}')
echo USDYOUR_CLOUD_PROVIDER_ACCESS_KEY
YOUR_CLOUD_PROVIDER_SECRET_KEY=USD(oc -n open-cluster-management-observability get secret <object-storage-secret> -o jsonpath="{.data.thanos\.yaml}" | base64 --decode | grep secret_key | awk '{print USD2}')
echo USDYOUR_CLOUD_PROVIDER_SECRET_KEY
observability-thanos-query (deployment) observability-thanos-compact (statefulset) observability-thanos-receive-default (statefulset) observability-thanos-rule (statefulset) observability-thanos-store-shard-x (statefulsets)
apiVersion: observability.open-cluster-management.io/v1beta2 kind: MultiClusterObservability metadata: name: observability spec: observabilityAddonSpec: {} storageConfig: metricObjectStorage: name: thanos-object-storage key: thanos.yaml
spec: advanced: compact: serviceAccountAnnotations: eks.amazonaws.com/role-arn: USDS3_ROLE store: serviceAccountAnnotations: eks.amazonaws.com/role-arn: USDS3_ROLE rule: serviceAccountAnnotations: eks.amazonaws.com/role-arn: USDS3_ROLE receive: serviceAccountAnnotations: eks.amazonaws.com/role-arn: USDS3_ROLE query: serviceAccountAnnotations: eks.amazonaws.com/role-arn: USDS3_ROLE
nodeSelector: node-role.kubernetes.io/infra: ""
apply -f multiclusterobservability_cr.yaml
get deploy multicluster-observability-operator -n open-cluster-management --show-labels
NAME READY UP-TO-DATE AVAILABLE AGE LABELS multicluster-observability-operator 1/1 1 1 35m installer.name=multiclusterhub,installer.namespace=open-cluster-management
labels: installer.name: multiclusterhub installer.namespace: open-cluster-management
thanos --version
spec: imagePullPolicy: Always imagePullSecret: multiclusterhub-operator-pull-secret observabilityAddonSpec: 1 enableMetrics: false 2 workers: 3
data: custom_rules.yaml: | groups: - name: cluster-health rules: - alert: ClusterCPUHealth-jb annotations: summary: Notify when CPU utilization on a cluster is greater than the defined utilization limit description: "The cluster has a high CPU usage: {{ USDvalue }} core for {{ USDlabels.cluster }} {{ USDlabels.clusterID }}." expr: | max(cluster:cpu_usage_cores:sum) by (clusterID, cluster, prometheus) > 0 for: 5s labels: cluster: "{{ USDlabels.cluster }}" prometheus: "{{ USDlabels.prometheus }}" severity: critical
data: custom_rules.yaml: | groups: - name: container-memory rules: - record: pod:container_memory_cache:sum expr: sum(container_memory_cache{pod!=""}) BY (pod, container)
get mco observability -o yaml
Observability components are deployed and running
apply -n open-cluster-management-observability -f observability-metrics-custom-allowlist.yaml
kind: ConfigMap apiVersion: v1 metadata: name: observability-metrics-custom-allowlist data: metrics_list.yaml: | names: 1 - node_memory_MemTotal_bytes rules: 2 - record: apiserver_request_duration_seconds:histogram_quantile_90 expr: histogram_quantile(0.90,sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\", verb!=\"WATCH\"}[5m])) by (verb,le))
kind: ConfigMap apiVersion: v1 metadata: name: observability-metrics-custom-allowlist namespace: test data: uwl_metrics_list.yaml: 1 names: 2 - sample_metrics
get mco observability -o yaml
-cluster_infrastructure_provider
apply -n open-cluster-management-observability -f observability-metrics-custom-allowlist.yaml
edit mco observability -o yaml
spec: advanced: retentionConfig: blockDuration: 2h deleteDelay: 48h retentionInLocal: 24h retentionResolutionRaw: 365d retentionResolution5m: 365d retentionResolution1h: 365d receive: resources: limits: memory: 4096Gi replicas: 3
edit mco observability
spec: advanced: retentionConfig: retentionResolutionRaw: 365d retentionResolution5m: 365d retentionResolution1h: 365d
collect_rules: - group: SNOResourceUsage annotations: description: > By default, a {sno} cluster does not collect pod and container resource metrics. Once a {sno} cluster reaches a level of resource consumption, these granular metrics are collected dynamically. When the cluster resource consumption is consistently less than the threshold for a period of time, collection of the granular metrics stops. selector: matchExpressions: - key: clusterType operator: In values: ["{sno}"] rules: - collect: SNOHighCPUUsage annotations: description: > Collects the dynamic metrics specified if the cluster cpu usage is constantly more than 70% for 2 minutes expr: (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100 > 70 for: 2m dynamic_metrics: names: - container_cpu_cfs_periods_total - container_cpu_cfs_throttled_periods_total - kube_pod_container_resource_limits - kube_pod_container_resource_requests - namespace_workload_pod:kube_pod_owner:relabel - node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate - node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate - collect: SNOHighMemoryUsage annotations: description: > Collects the dynamic metrics specified if the cluster memory usage is constantly more than 70% for 2 minutes expr: (1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(kube_node_status_allocatable{resource=\"memory\"})) * 100 > 70 for: 2m dynamic_metrics: names: - kube_pod_container_resource_limits - kube_pod_container_resource_requests - namespace_workload_pod:kube_pod_owner:relabel matches: - __name__="container_memory_cache",container!="" - __name__="container_memory_rss",container!="" - __name__="container_memory_swap",container!="" - __name__="container_memory_working_set_bytes",container!=""
collect_rules: - group: -SNOResourceUsage
spec: advanced: receive: replicas: 6
create secret generic <tls_secret_name> --from-file=ca.crt=<path_to_file> -n open-cluster-management-observability
apiVersion: v1 kind: Secret metadata: name: <tls_secret_name> namespace: open-cluster-management-observability type: Opaque data: ca.crt: <base64_encoded_ca_certificate>
edit mco observability -o yaml
metricObjectStorage: key: thanos.yaml name: thanos-object-storage tlsSecretName: tls-certs-secret 1 tlsSecretMountPath: /etc/minio/certs 2
thanos.yaml: | type: s3 config: bucket: "thanos" endpoint: "minio:9000" insecure: false 1 access_key: "minio" secret_key: "minio123" http_config: tls_config: ca_file: /etc/minio/certs/ca.crt 2 insecure_skip_verify: false
thanos.yaml: | type: s3 config: bucket: "thanos" endpoint: "minio:9000" insecure: false access_key: "minio" secret_key: "minio123" http_config: tls_config: ca_file: /etc/minio/certs/ca.crt 1 cert_file: /etc/minio/certs/public.crt key_file: /etc/minio/certs/private.key insecure_skip_verify: false
-n open-cluster-management-observability get pods -l app.kubernetes.io/name=thanos-store
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: AddOnDeploymentConfig metadata: name: <addon-deploy-config-name> namespace: <managed-cluster-name> spec: agentInstallNamespace: open-cluster-management-addon-observability proxyConfig: httpsProxy: "http://<username>:<password>@<ip>:<port>" 1 noProxy: ".cluster.local,.svc,172.30.0.1" 2
-n default describe svc kubernetes | grep IP:
apiVersion: addon.open-cluster-management.io/v1alpha1 kind: ManagedClusterAddOn metadata: name: observability-controller namespace: <managed-cluster-name> spec: installNamespace: open-cluster-management-addon-observability configs: - group: addon.open-cluster-management.io resource: addondeploymentconfigs name: <addon-deploy-config-name> namespace: <managed-cluster-name>
spec: advanced: customObservabilityHubURL: <yourURL> customAlertmanagerHubURL: <yourURL>
apiVersion: route.openshift.io/v1 kind: Route metadata: name: proxy-observatorium-api namespace: open-cluster-management-observability spec: host: <intermediate_component_url> port: targetPort: public tls: insecureEdgeTerminationPolicy: None termination: passthrough to: kind: Service name: observability-observatorium-api weight: 100 wildcardPolicy: None
apiVersion: route.openshift.io/v1 kind: Route metadata: name: alertmanager-proxy namespace: open-cluster-management-observability spec: host: <intermediate_component_url> path: /api/v2 port: targetPort: oauth-proxy tls: insecureEdgeTerminationPolicy: Redirect termination: reencrypt to: kind: Service name: alertmanager weight: 100 wildcardPolicy: None
apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: awesome-app-metrics-role rules: - apiGroups: - "cluster.open-cluster-management.io" resources: - managedclusters: 1 resourceNames: 2 - devcluster1 - devcluster2 verbs: 3 - metrics/AwesomeAppNS
kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: awesome-app-metrics-role-binding subjects: - kind: Group apiGroup: rbac.authorization.k8s.io name: my-awesome-app-admins roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: awesome-app-metrics-role
get route rbac-query-proxy -n open-cluster-management-observability
MY_TOKEN=USD(oc whoami --show-token)
-n openshift-ingress get secret router-certs-default -o jsonpath="{.data.tls\.crt}" | base64 -d > ca.crt
-n open-cluster-management-observability create secret tls proxy-byo-ca --cert ./ca.crt --key ./ca.key
curl --cacert ./ca.crt -H "Authorization: Bearer {TOKEN}" https://{PROXY_ROUTE_URL}/api/v1/query?query={QUERY_EXPRESSION}
apiVersion: v1 kind: Secret metadata: name: victoriametrics namespace: open-cluster-management-observability type: Opaque stringData: ep.yaml: | 1 url: http://victoriametrics:8428/api/v1/write 2 http_client_config: 3 basic_auth: 4 username: test 5 password: test 6 tls_config: 7 secret_name: 8 ca_file_key: 9 cert_file_key: 10 key_file_key: 11 insecure_skip_verify: 12
spec: storageConfig: writeStorage: 1 - key: ep.yaml name: victoriametrics
./setup-grafana-dev.sh --deploy secret/grafana-dev-config created deployment.apps/grafana-dev created service/grafana-dev created serviceaccount/grafana-dev created clusterrolebinding.rbac.authorization.k8s.io/open-cluster-management:grafana-crb-dev created route.route.openshift.io/grafana-dev created persistentvolumeclaim/grafana-dev created oauthclient.oauth.openshift.io/grafana-proxy-client-dev created deployment.apps/grafana-dev patched service/grafana-dev patched route.route.openshift.io/grafana-dev patched oauthclient.oauth.openshift.io/grafana-proxy-client-dev patched clusterrolebinding.rbac.authorization.k8s.io/open-cluster-management:grafana-crb-dev patched
./switch-to-grafana-admin.sh kube:admin User <kube:admin> switched to be grafana admin
grafana-cli
./generate-dashboard-configmap-yaml.sh "Your Dashboard Name" Save dashboard <your-dashboard-name> to ./your-dashboard-name.yaml
kind: ConfigMap apiVersion: v1 metadata: name: USDyour-dashboard-name namespace: open-cluster-management-observability labels: grafana-custom-dashboard: "true" data: USDyour-dashboard-name.json: |- USDyour_dashboard_json
./generate-dashboard-configmap-yaml.sh "Demo Dashboard"
Save dashboard <demo-dashboard> to ./demo-dashboard.yaml
annotations: observability.open-cluster-management.io/dashboard-folder: Custom
apply -f demo-dashboard.yaml
./setup-grafana-dev.sh --clean secret "grafana-dev-config" deleted deployment.apps "grafana-dev" deleted serviceaccount "grafana-dev" deleted route.route.openshift.io "grafana-dev" deleted persistentvolumeclaim "grafana-dev" deleted oauthclient.oauth.openshift.io "grafana-proxy-client-dev" deleted clusterrolebinding.rbac.authorization.k8s.io "open-cluster-management:grafana-crb-dev" deleted
data: managed_cluster.yaml: | ignore_labels: 1 - clusterID - cluster.open-cluster-management.io/clusterset - feature.open-cluster-management.io/addon-application-manager - feature.open-cluster-management.io/addon-cert-policy-controller - feature.open-cluster-management.io/addon-cluster-proxy - feature.open-cluster-management.io/addon-config-policy-controller - feature.open-cluster-management.io/addon-governance-policy-framework - feature.open-cluster-management.io/addon-iam-policy-controller - feature.open-cluster-management.io/addon-observability-controller - feature.open-cluster-management.io/addon-search-collector - feature.open-cluster-management.io/addon-work-manager - installer.name - installer.namespace - local-cluster - name labels: 2 - cloud - vendor
data: managed_cluster.yaml: | ignore_labels: - clusterID - cluster.open-cluster-management.io/clusterset - feature.open-cluster-management.io/addon-application-manager - feature.open-cluster-management.io/addon-cert-policy-controller - feature.open-cluster-management.io/addon-cluster-proxy - feature.open-cluster-management.io/addon-config-policy-controller - feature.open-cluster-management.io/addon-governance-policy-framework - feature.open-cluster-management.io/addon-iam-policy-controller - feature.open-cluster-management.io/addon-observability-controller - feature.open-cluster-management.io/addon-search-collector - feature.open-cluster-management.io/addon-work-manager - installer.name - installer.namespace - local-cluster - name labels: - cloud - department - vendor
data: managed_cluster.yaml: | ignore_labels: - clusterID - installer.name - installer.namespace labels: - cloud - vendor - local-cluster - name
enabled managedcluster labels: <label>
data: managed_cluster.yaml: | ignore_labels: - clusterID - installer.name - installer.namespace - local-cluster - name labels: - cloud - vendor
disabled managedcluster label: <label>
-n open-cluster-management-observability get secret alertmanager-config --template='{{ index .data "alertmanager.yaml" }}' |base64 -d > alertmanager.yaml
-n open-cluster-management-observability create secret generic alertmanager-config --from-file=alertmanager.yaml --dry-run -o=yaml | oc -n open-cluster-management-observability replace secret --filename=-
global smtp_smarthost: 'localhost:25' smtp_from: 'alertmanager@example.org' smtp_auth_username: 'alertmanager' smtp_auth_password: 'password' templates: - '/etc/alertmanager/template/*.tmpl' route: group_by: ['alertname', 'cluster', 'service'] group_wait: 30s group_interval: 5m repeat_interval: 3h receiver: team-X-mails routes: - match_re: service: ^(foo1|foo2|baz)USD receiver: team-X-mails
create secret tls tls --cert=</path/to/cert.crt> --key=</path/to/cert.key> -n open-cluster-management-observability
advanced: alertmanager: secrets: ['tls']
tls_config: cert_file: '/etc/alertmanager/secrets/tls/tls.crt' key_file: '/etc/alertmanager/secrets/tls/tls.key'
-n open-cluster-management-observability get secret alertmanager-config --template='{{ index .data "alertmanager.yaml" }}' |base64 -d > alertmanager.yaml
"global": "http_config": "tls_config": "cert_file": "/etc/alertmanager/secrets/storyverify/tls.crt" "key_file": "/etc/alertmanager/secrets/storyverify/tls.key"
-n open-cluster-management-observability create secret generic alertmanager-config --from-file=alertmanager.yaml --dry-run -o=yaml
-n open-cluster-management-observability replace secret --filename=-
global: slack_api_url: '<slack_webhook_url>' route: receiver: 'slack-notifications' group_by: [alertname, datacenter, app] receivers: - name: 'slack-notifications' slack_configs: - channel: '#alerts' text: 'https://internal.myorg.net/wiki/alerts/{{ .GroupLabels.app }}/{{ .GroupLabels.alertname }}'
global: slack_api_url: '<slack_webhook_url>' http_config: proxy_url: http://****
metadata: annotations: mco-disable-alerting: "true"
amtool silence add --alertmanager.url="http://localhost:9093" --author="user" --comment="Silencing sample alert" alertname="SampleAlert"
amtool silence add --alertmanager.url="http://localhost:9093" --author="user" --comment="Silencing sample alert" <match-label-1>=<match-value-1> <match-label-2>=<match-value-2>
amtool silence add --alertmanager.url="http://localhost:9093" --author="user" --comment="Silencing sample alert" --duration="1h" alertname="SampleAlert"
amtool silence add --alertmanager.url="http://localhost:9093" --author="user" --comment="Silencing sample alert" --start="2023-04-14T15:04:05-07:00" alertname="SampleAlert"
amtool silence --alertmanager.url="http://localhost:9093"
amtool silence expire --alertmanager.url="http://localhost:9093" "d839aca9-ed46-40be-84c4-dca8773671da"
amtool silence expire --alertmanager.url="http://localhost:9093" USD(amtool silence query --alertmanager.url="http://localhost:9093" -q)
global: resolve_timeout: 1h inhibit_rules: 1 - equal: - namespace source_match: 2 severity: critical target_match_re: severity: warning|info
ALERTS{alertname="foo", namespace="ns-1", severity="critical"} ALERTS{alertname="foo", namespace="ns-1", severity="warning"}
ALERTS{alertname="foo", namespace="ns-1", severity="critical"} ALERTS{alertname="foo", namespace="ns-2", severity="warning"}
amtool alert --alertmanager.url="http://localhost:9093" --inhibited
get policyreport --all-namespaces