Repository: conbench/conbench (language: python)

Issue:
kube-prometheus stack: inject custom grafana admin credentials
For local dev, anonymous access is nice:
https://github.com/conbench/conbench/blob/016fc318b50ae90549c231658ef443836fc9f932/k8s/kube-prometheus/conbench-flavor.jsonnet#L38

For more serious usage, build a pragmatic way to set credentials via k8s secrets/config map.


Discussion / hints:
Potentially useful command from https://github.com/prometheus-operator/kube-prometheus/discussions/1578
```
GRAFANA_POD=$(kubectl get pods --selector=app=grafana -n monitoring | tail -n 1 | awk '{print $1}')
kubectl exec --stdin --namespace="monitoring" --tty $GRAFANA_POD -- /bin/bash -c "grafana-cli admin reset-admin-password $GRAFANA_ADMIN_PASSWORD"
```
Changing selector, that works:
```
GRAFANA_POD=$(kubectl get pods --selector=app.kubernetes.io/name=grafana -n monitoring | tail -n 1 | awk '{print $1}')
```
and then this worked:
```
$ kubectl exec --stdin --namespace="monitoring" --tty $GRAFANA_POD -- /bin/bash -c "grafana-cli admin reset-admin-password $GRAFANA_ADMIN_PASSWORD"
INFO [03-21|15:58:31] Starting Grafana                         logger=settings version= commit= branch= compiled=1970-01-01T00:00:00Z
INFO [03-21|15:58:31] Config loaded from                       logger=settings file=/usr/share/grafana/conf/defaults.ini
INFO [03-21|15:58:31] Config overridden from Environment variable logger=settings var="GF_PATHS_DATA=/var/lib/grafana"
INFO [03-21|15:58:31] Config overridden from Environment variable logger=settings var="GF_PATHS_LOGS=/var/log/grafana"
INFO [03-21|15:58:31] Config overridden from Environment variable logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins"
INFO [03-21|15:58:31] Config overridden from Environment variable logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning"
INFO [03-21|15:58:31] Path Home                                logger=settings path=/usr/share/grafana
INFO [03-21|15:58:31] Path Data                                logger=settings path=/var/lib/grafana
INFO [03-21|15:58:31] Path Logs                                logger=settings path=/var/log/grafana
INFO [03-21|15:58:31] Path Plugins                             logger=settings path=/var/lib/grafana/plugins
INFO [03-21|15:58:31] Path Provisioning                        logger=settings path=/etc/grafana/provisioning
INFO [03-21|15:58:31] App mode production                      logger=settings
INFO [03-21|15:58:31] Envelope encryption state                logger=secrets enabled=true current provider=secretKey.v1

Admin password changed successfully ✔
```


Fix patch:
diff --git a/Makefile b/Makefile
index eec7835cb..03c2a44d4 100644
--- a/Makefile
+++ b/Makefile
@@ -284,22 +284,31 @@ set-build-info:
 # coreos/jsonnet-ci container image used below comes with `jq` (one could
 # install this locally with e.g. sudo dnf install jsonnet) and also with
 # gojsontoyaml (which is where I resorted to looking for a container image that
-# has all dependencies baked in).
+# has all dependencies baked in). If MUTATE_JSONNET_FILE_FOR_MINIKUBE is set:
+# do modifications for CI and local dev on minikube. Modify JSONNET file. Maybe
+# it's chaotic to do sed-based templating on top of JSONNET, we can maybe do
+# that better in the future based on JSONNET external variables?
 .PHONY: jsonnet-kube-prom-manifests
 jsonnet-kube-prom-manifests:
 #	rm -rf _kpbuild
 	mkdir -p _kpbuild && cd _kpbuild  && mkdir -p cb-kube-prometheus
 	cd _kpbuild/cb-kube-prometheus && \
 		docker run --user $$(id -u):$$(id -g) --rm -v $$(pwd):$$(pwd) --workdir $$(pwd) quay.io/coreos/jsonnet-ci \
-			jb init || echo "exists"
+			jb init || echo "exists: ignore, proceed"
 	cd _kpbuild/cb-kube-prometheus && \
 		time docker run --user $$(id -u):$$(id -g) --rm -v $$(pwd):$$(pwd) --workdir $$(pwd) quay.io/coreos/jsonnet-ci \
 			jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@v0.12.0
-	cd _kpbuild/cb-kube-prometheus && \
-		wget https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/v0.12.0/build.sh -O build.sh
 	cp k8s/kube-prometheus/conbench-flavor.jsonnet _kpbuild/cb-kube-prometheus
 	cp k8s/kube-prometheus/conbench-grafana-dashboard.json _kpbuild/cb-kube-prometheus
 	cp k8s/kube-prometheus/kube-prom-no-req-no-lim.jsonnet _kpbuild/cb-kube-prometheus
+	@if [ -z "$${MUTATE_JSONNET_FILE_FOR_MINIKUBE:=}" ]; then \
+			echo "MUTATE_JSONNET_FILE_FOR_MINIKUBE not set"; \
+		else \
+			echo "MUTATE_JSONNET_FILE_FOR_MINIKUBE set, mutate JSONNET"; \
+			sed -i.bak 's|// (import "kube-prom|(import "kube-prom|g' _kpbuild/cb-kube-prometheus/conbench-flavor.jsonnet; \
+			sed -i.bak 's|// "auth.anonymous"|"auth.anonymous"|g' _kpbuild/cb-kube-prometheus/conbench-flavor.jsonnet; \
+			cat _kpbuild/cb-kube-prometheus/conbench-flavor.jsonnet; \
+		fi
 	@if [ -z "$${PROM_REMOTE_WRITE_ENDPOINT_URL:=}" ]; then \
 			echo "PROM_REMOTE_WRITE_ENDPOINT_URL not set"; \
 		else \
@@ -314,6 +323,8 @@ jsonnet-kube-prom-manifests:
 			sed -i.bak "s|PROM_REMOTE_WRITE_CLUSTER_LABEL_VALUE|$${PROM_REMOTE_WRITE_CLUSTER_LABEL_VALUE}|g" \
 				_kpbuild/cb-kube-prometheus/conbench-flavor.jsonnet; \
 		fi
+	cd _kpbuild/cb-kube-prometheus && \
+		wget https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/v0.12.0/build.sh -O build.sh
 	cd _kpbuild/cb-kube-prometheus && \
 		time docker run --user $$(id -u):$$(id -g) --rm -v $$(pwd):$$(pwd) --workdir $$(pwd) quay.io/coreos/jsonnet-ci \
 			bash build.sh conbench-flavor.jsonnet
diff --git a/k8s/kube-prometheus/conbench-flavor.jsonnet b/k8s/kube-prometheus/conbench-flavor.jsonnet
index 98a5c1298..5f464877a 100644
--- a/k8s/kube-prometheus/conbench-flavor.jsonnet
+++ b/k8s/kube-prometheus/conbench-flavor.jsonnet
@@ -14,7 +14,7 @@ local kp =
   // (import 'kube-prometheus/addons/custom-metrics.libsonnet') +
   // (import 'kube-prometheus/addons/external-metrics.libsonnet') +
   // (import 'kube-prometheus/addons/pyrra.libsonnet') +
-  (import 'kube-prom-no-req-no-lim.jsonnet') +
+  // (import "kube-prom-no-req-no-lim.jsonnet") +  // auto-uncommented-by-ci
   {
     values+:: {
       prometheus+: {
@@ -35,7 +35,7 @@ local kp =
         config: {
           // http://docs.grafana.org/installation/configuration/
           sections: {
-            'auth.anonymous': { enabled: true },
+            // "auth.anonymous": { enabled: true },  // auto-uncommented-by-ci
             // Configure Grafana to be available under sub path instead of
             // root.
             server: {
@@ -59,7 +59,7 @@ local kp =
             // If left as-is then Prometheus starts and periodically shows
             // a log line saying that this is an invalid URL. That's fine!
             // That is: replace this, if you want the k8s cluster to send
-            // metrics ot an external system. Leave this as-is when this
+            // metrics to an external system. Leave this as-is when this
             // capabily is not needed.
             url: 'PROM_REMOTE_WRITE_ENDPOINT_URL',
             basicAuth: {


Test patch:
diff --git a/ci/minikube/test-conbench-on-mk.sh b/ci/minikube/test-conbench-on-mk.sh
index fa781cb05..a42b813f2 100644
--- a/ci/minikube/test-conbench-on-mk.sh
+++ b/ci/minikube/test-conbench-on-mk.sh
@@ -93,7 +93,11 @@ EOF
 
 export PROM_REMOTE_WRITE_CLUSTER_LABEL_VALUE="ci-conbench-on-$(hostname -s)"
 
-# Build custom version of kube-prometheus stack.
+
+# JSONNET-build our custom version of kube-prometheus. Before JSONNET
+# compilation mutate the main document to make adjustments for the minikube
+# environment (smaller resource footprint, anonymous access to Grafana UI).
+export MUTATE_JSONNET_FILE_FOR_MINIKUBE=true
 ( cd "${CONBENCH_REPO_ROOT_DIR}" && make jsonnet-kube-prom-manifests )
 
 # Set up the kube-prometheus stack. This follows the customization instructions