Repository: open-telemetry/opentelemetry-operator

Issue:
Upgrade prometheus and prometheus-operator dependencies to latest
### Component(s)

target allocator

### Describe the issue you're reporting

There are currently several issues blocking us from upgrading to the latest version of prometheus dependencies:

- [x] Some changes to logging that we should be able to easily make.
- [x] Prometheus-operator now generates Prometheus config a bit differently, and we'll need to clean up our handling of namespace selectors in target allocator to be compatible.
- [ ] Prometheus 0.300.0 includes a breaking change to default scrape protocols, and this change is supported by the Otel collector starting in version 0.120.0. We need to account for this in the configurations generated by target allocator and think about when and how we want to introduce the breaking change.



Discussion:
@oldium: Is it planned to fix the following config or should I create a separate issue for it?

```
    service:
      telemetry:
        metrics:
          level: detailed
          readers:
            - pull:
                exporter:
                  prometheus:
                    host: '0.0.0.0'
                    port: 8888
 ```

This creates an “address already in use” Collector startup error, because OTEL K8s Operator adds a default `service.telemetry.metrics.address=0.0.0.0:8888`, which conflicts with custom `service.telemetry.metrics.readers` config.@iblancasa: > Is it planned to fix the following config or should I create a separate issue for it?
> 
> ```
>     service:
>       telemetry:
>         metrics:
>           level: detailed
>           readers:
>             - pull:
>                 exporter:
>                   prometheus:
>                     host: '0.0.0.0'
>                     port: 8888
> ```
> 
> This creates an “address already in use” Collector startup error, because OTEL K8s Operator adds a default `service.telemetry.metrics.address=0.0.0.0:8888`, which conflicts with custom `service.telemetry.metrics.readers` config.

Please, create something separate. That is not related to this ticket.

Solution patch:
diff --git a/cmd/otel-allocator/internal/watcher/promOperator.go b/cmd/otel-allocator/internal/watcher/promOperator.go
index cb85fdc0..deaa9a85 100644
--- a/cmd/otel-allocator/internal/watcher/promOperator.go
+++ b/cmd/otel-allocator/internal/watcher/promOperator.go
@@ -83,6 +83,7 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat
 				ProbeNamespaceSelector:          cfg.PrometheusCR.ProbeNamespaceSelector,
 				ServiceDiscoveryRole:            &serviceDiscoveryRole,
 			},
+			EvaluationInterval: monitoringv1.Duration("30s"),
 		},
 	}
 
@@ -133,6 +134,7 @@ func NewPrometheusCRWatcher(ctx context.Context, logger logr.Logger, cfg allocat
 		probeNamespaceSelector:          cfg.PrometheusCR.ProbeNamespaceSelector,
 		resourceSelector:                resourceSelector,
 		store:                           store,
+		prometheusCR:                    prom,
 	}, nil
 }
 
@@ -152,6 +154,7 @@ type PrometheusCRWatcher struct {
 	probeNamespaceSelector          *metav1.LabelSelector
 	resourceSelector                *prometheus.ResourceSelector
 	store                           *assets.StoreBuilder
+	prometheusCR                    *monitoringv1.Prometheus
 }
 
 func getNamespaceInformer(ctx context.Context, allowList, denyList map[string]struct{}, promOperatorLogger *slog.Logger, clientset kubernetes.Interface, operatorMetrics *operator.Metrics) (cache.SharedIndexInformer, error) {
@@ -367,13 +370,7 @@ func (w *PrometheusCRWatcher) LoadConfig(ctx context.Context) (*promconfig.Confi
 		}
 
 		generatedConfig, err := w.configGenerator.GenerateServerConfiguration(
-			"30s",
-			"",
-			nil,
-			nil,
-			&monitoringv1.TSDBSpec{},
-			nil,
-			nil,
+			w.prometheusCR,
 			serviceMonitorInstances,
 			podMonitorInstances,
 			probeInstances,
diff --git a/go.mod b/go.mod
index 78cf3c2c..2340fd45 100644
--- a/go.mod
+++ b/go.mod
@@ -25,9 +25,9 @@ require (
 	github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e
 	github.com/operator-framework/api v0.30.0
 	github.com/operator-framework/operator-lib v0.17.0
-	github.com/prometheus-operator/prometheus-operator v0.77.2
-	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.77.2
-	github.com/prometheus-operator/prometheus-operator/pkg/client v0.77.2
+	github.com/prometheus-operator/prometheus-operator v0.78.2
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.78.2
+	github.com/prometheus-operator/prometheus-operator/pkg/client v0.78.2
 	github.com/prometheus/client_golang v1.21.0
 	github.com/prometheus/common v0.63.0
 	github.com/prometheus/prometheus v0.55.1
diff --git a/go.sum b/go.sum
index c87b40e3..e0dae942 100644
--- a/go.sum
+++ b/go.sum
@@ -537,12 +537,12 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/prometheus-community/prom-label-proxy v0.11.0 h1:IO02WiiFMfcIqvjhwMbCYnDJiTNcSHBrkCGRQ/7KDd0=
 github.com/prometheus-community/prom-label-proxy v0.11.0/go.mod h1:lfvrG70XqsxWDrSh1843QXBG0fSg8EbIXmAo8xGsvw8=
-github.com/prometheus-operator/prometheus-operator v0.77.2 h1:HRewl8m49URAOfRTZql2YjjNGZ8+25srcGji5wx/tic=
-github.com/prometheus-operator/prometheus-operator v0.77.2/go.mod h1:y2XLK9xQ4ulaAtWmvKDM+hQNl7w2Wv/iE+tuF5Xnb8A=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.77.2 h1:F/MALZ518KfI1zEg+Kg8/uTzoXKDyqw+LNC/5irJlJE=
-github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.77.2/go.mod h1:D0KY8md81DQKdaR/cXwnhoWB3MYYyc/UjvqE8GFkIvA=
-github.com/prometheus-operator/prometheus-operator/pkg/client v0.77.2 h1:rEBVlnqBSkNc6ufXnDu/GtSyS2jL1l6imtufzJUfOXI=
-github.com/prometheus-operator/prometheus-operator/pkg/client v0.77.2/go.mod h1:Q+R7L3DIlJ2a8R+W+7wjk0B25Ci/VTy7V07Zvq327x0=
+github.com/prometheus-operator/prometheus-operator v0.78.2 h1:ismQgRxQbfZf/iFpEXxpbciqmzpUKmQLKmdJl+2k7KE=
+github.com/prometheus-operator/prometheus-operator v0.78.2/go.mod h1:6Vv58mZOaqQ2TLLXskA1e7RnQUwmtLIpV5ggHx7rb08=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.78.2 h1:SyoVBXD/r0PntR1rprb90ClI32FSUNOCWqqTatnipHM=
+github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.78.2/go.mod h1:SvsRXw4m1F2vk7HquU5h475bFpke27mIUswfyw9u3ug=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.78.2 h1:kiCXsZol0PCuSKA/ShGOzwSqEkAXSeQFLKLrra/vOSk=
+github.com/prometheus-operator/prometheus-operator/pkg/client v0.78.2/go.mod h1:gw/WDyY7l0PLuPHdQRuAaibYBBKlwZSQkO9MU6M2MPw=
 github.com/prometheus/alertmanager v0.27.0 h1:V6nTa2J5V4s8TG4C4HtrBP/WNSebCCTYGGv4qecA/+I=
 github.com/prometheus/alertmanager v0.27.0/go.mod h1:8Ia/R3urPmbzJ8OsdvmZvIprDwvwmYCmUbwBL+jlPOE=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=



Test patch:
diff --git a/cmd/otel-allocator/internal/watcher/promOperator_test.go b/cmd/otel-allocator/internal/watcher/promOperator_test.go
index cb87bfa4..2914d2d8 100644
--- a/cmd/otel-allocator/internal/watcher/promOperator_test.go
+++ b/cmd/otel-allocator/internal/watcher/promOperator_test.go
@@ -1251,6 +1251,7 @@ func getTestPrometheusCRWatcher(
 				ScrapeConfigNamespaceSelector:   cfg.PrometheusCR.ScrapeConfigNamespaceSelector,
 				ServiceDiscoveryRole:            &serviceDiscoveryRole,
 			},
+			EvaluationInterval: monitoringv1.Duration("30s"),
 		},
 	}
 
@@ -1295,6 +1296,7 @@ func getTestPrometheusCRWatcher(
 		scrapeConfigNamespaceSelector:   cfg.PrometheusCR.ScrapeConfigNamespaceSelector,
 		resourceSelector:                resourceSelector,
 		store:                           store,
+		prometheusCR:                    prom,
 	}, source
 
 }