Repository: open-telemetry/opentelemetry-operator

Issue:
[target allocator] Use Otel Go SDK for metrics reporting
### Component(s)

target allocator

### Is your feature request related to a problem? Please describe.

Target Allocator uses prometheus metrics for metric reporting.

### Describe the solution you'd like

Consider moving to otel go SDK.

### Describe alternatives you've considered

_No response_

### Additional context

_No response_

Discussion:
@swiatekm: I'm definitely in favor of doing this, and in general improving the internal observability of the operator. I'd love to be able to trace reconciliation and webhook calls, for example. Logging is also a bit of a mess, and standardizing on the Otel SDK would help that.

Solution patch:
diff --git a/cmd/otel-allocator/internal/allocation/allocator.go b/cmd/otel-allocator/internal/allocation/allocator.go
index 73d764df..ebd4ffe7 100644
--- a/cmd/otel-allocator/internal/allocation/allocator.go
+++ b/cmd/otel-allocator/internal/allocation/allocator.go
@@ -4,13 +4,17 @@
 package allocation
 
 import (
+	"context"
 	"errors"
 	"runtime"
 	"slices"
 	"sync"
+	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 
 	"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/diff"
 	"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/target"
@@ -25,19 +29,48 @@ import (
 
 var _ Allocator = &allocator{}
 
-func newAllocator(log logr.Logger, strategy Strategy, opts ...Option) Allocator {
+func newAllocator(log logr.Logger, strategy Strategy, opts ...Option) (Allocator, error) {
+	meter := otel.GetMeterProvider().Meter("targetallocator")
+	// targetsPerCollector records how many targets have been assigned to each collector.
+	// It is currently the responsibility of the strategy to track this information.
+	targetsPerCollector, err := meter.Int64Gauge("opentelemetry_allocator_targets_per_collector", metric.WithDescription("The number of targets for each collector."))
+	if err != nil {
+		return nil, err
+	}
+	collectorsAllocatable, err := meter.Int64Gauge("opentelemetry_allocator_collectors_allocatable", metric.WithDescription("Number of collectors the allocator is able to allocate to."))
+	if err != nil {
+		return nil, err
+	}
+	timeToAssign, err := meter.Float64Histogram("opentelemetry_allocator_time_to_allocate", metric.WithDescription("The time it takes to allocate"))
+	if err != nil {
+		return nil, err
+	}
+	targetsRemaining, err := meter.Int64Gauge("opentelemetry_allocator_targets_remaining", metric.WithDescription("Number of targets kept after filtering."))
+	if err != nil {
+		return nil, err
+	}
+	targetsUnassigned, err := meter.Int64Gauge("opentelemetry_allocator_targets_unassigned", metric.WithDescription("Number of targets that could not be assigned due to missing node label."))
+	if err != nil {
+		return nil, err
+	}
+
 	chAllocator := &allocator{
 		strategy:                      strategy,
 		collectors:                    make(map[string]*Collector),
 		targetItems:                   make(map[target.ItemHash]*target.Item),
 		targetItemsPerJobPerCollector: make(map[string]map[string]map[target.ItemHash]bool),
 		log:                           log,
+		targetsPerCollector:           targetsPerCollector,
+		collectorsAllocatable:         collectorsAllocatable,
+		timeToAssign:                  timeToAssign,
+		targetsRemaining:              targetsRemaining,
+		targetsUnassigned:             targetsUnassigned,
 	}
 	for _, opt := range opts {
 		opt(chAllocator)
 	}
 
-	return chAllocator
+	return chAllocator, nil
 }
 
 type allocator struct {
@@ -59,7 +92,12 @@ type allocator struct {
 
 	log logr.Logger
 
-	filter Filter
+	filter                Filter
+	targetsPerCollector   metric.Int64Gauge
+	collectorsAllocatable metric.Int64Gauge
+	timeToAssign          metric.Float64Histogram
+	targetsRemaining      metric.Int64Gauge
+	targetsUnassigned     metric.Int64Gauge
 }
 
 // SetFilter sets the filtering hook to use.
@@ -76,13 +114,16 @@ func (a *allocator) SetFallbackStrategy(strategy Strategy) {
 // load balancing decisions. This method should be called when there are
 // new targets discovered or existing targets are shutdown.
 func (a *allocator) SetTargets(targets []*target.Item) {
-	timer := prometheus.NewTimer(TimeToAssign.WithLabelValues("SetTargets", a.strategy.GetName()))
-	defer timer.ObserveDuration()
+	begin := time.Now()
+	defer func() {
+		a.timeToAssign.Record(context.Background(), time.Since(begin).Seconds(), metric.WithAttributes(attribute.String("method", "SetTargets"), attribute.String("strategy", a.strategy.GetName())))
+	}()
 
 	if a.filter != nil {
 		targets = a.filter.Apply(targets)
 	}
-	RecordTargetsKept(targets)
+
+	a.targetsRemaining.Record(context.Background(), int64(len(targets)))
 	concurrency := runtime.NumCPU() * 2 // determined experimentally
 	targetMap := buildTargetMap(targets, concurrency)
 
@@ -100,10 +141,12 @@ func (a *allocator) SetTargets(targets []*target.Item) {
 // SetCollectors sets the set of collectors with key=collectorName, value=Collector object.
 // This method is called when Collectors are added or removed.
 func (a *allocator) SetCollectors(collectors map[string]*Collector) {
-	timer := prometheus.NewTimer(TimeToAssign.WithLabelValues("SetCollectors", a.strategy.GetName()))
-	defer timer.ObserveDuration()
+	begin := time.Now()
+	defer func() {
+		a.timeToAssign.Record(context.Background(), time.Since(begin).Seconds(), metric.WithAttributes(attribute.String("strategy", a.strategy.GetName()), attribute.String("method", "SetCollectors")))
+	}()
 
-	CollectorsAllocatable.WithLabelValues(a.strategy.GetName()).Set(float64(len(collectors)))
+	a.collectorsAllocatable.Record(context.Background(), int64(len(collectors)), metric.WithAttributes(attribute.String("strategy", a.strategy.GetName())))
 	if len(collectors) == 0 {
 		a.log.Info("No collector instances present")
 	}
@@ -193,7 +236,7 @@ func (a *allocator) handleTargets(diff diff.Changes[target.ItemHash, *target.Ite
 	if unassignedTargets > 0 {
 		err := errors.Join(assignmentErrors...)
 		a.log.Info("Could not assign targets for some jobs", "targets", unassignedTargets, "error", err)
-		TargetsUnassigned.Set(float64(unassignedTargets))
+		a.targetsUnassigned.Record(context.Background(), int64(unassignedTargets))
 	}
 }
 
@@ -218,8 +261,7 @@ func (a *allocator) addTargetToTargetItems(tg *target.Item) error {
 	tg.CollectorName = colOwner.Name
 	a.addCollectorTargetItemMapping(tg)
 	a.collectors[colOwner.Name].NumTargets++
-	TargetsPerCollector.WithLabelValues(colOwner.String(), a.strategy.GetName()).Set(float64(a.collectors[colOwner.String()].NumTargets))
-
+	a.targetsPerCollector.Record(context.Background(), int64(a.collectors[colOwner.String()].NumTargets), metric.WithAttributes(attribute.String("collector_name", colOwner.String()), attribute.String("strategy", a.strategy.GetName())))
 	return nil
 }
 
@@ -234,7 +276,7 @@ func (a *allocator) unassignTargetItem(item *target.Item) {
 		return
 	}
 	c.NumTargets--
-	TargetsPerCollector.WithLabelValues(item.CollectorName, a.strategy.GetName()).Set(float64(c.NumTargets))
+	a.targetsPerCollector.Record(context.Background(), int64(c.NumTargets), metric.WithAttributes(attribute.String("collector_name", item.CollectorName), attribute.String("strategy", a.strategy.GetName())))
 	delete(a.targetItemsPerJobPerCollector[item.CollectorName][item.JobName], item.Hash())
 	if len(a.targetItemsPerJobPerCollector[item.CollectorName][item.JobName]) == 0 {
 		delete(a.targetItemsPerJobPerCollector[item.CollectorName], item.JobName)
@@ -258,7 +300,7 @@ func (a *allocator) removeCollector(collector *Collector) {
 		}
 	}
 	delete(a.targetItemsPerJobPerCollector, collector.Name)
-	TargetsPerCollector.WithLabelValues(collector.Name, a.strategy.GetName()).Set(0)
+	a.targetsPerCollector.Record(context.Background(), 0, metric.WithAttributes(attribute.String("collector_name", collector.Name), attribute.String("strategy", a.strategy.GetName())))
 }
 
 // addCollectorTargetItemMapping keeps track of which collector has which jobs and targets
@@ -304,7 +346,7 @@ func (a *allocator) handleCollectors(diff diff.Changes[string, *Collector]) {
 	if unassignedTargets > 0 {
 		err := errors.Join(assignmentErrors...)
 		a.log.Info("Could not assign targets for some jobs", "targets", unassignedTargets, "error", err)
-		TargetsUnassigned.Set(float64(unassignedTargets))
+		a.targetsUnassigned.Record(context.Background(), int64(unassignedTargets))
 	}
 }
 
diff --git a/cmd/otel-allocator/internal/allocation/strategy.go b/cmd/otel-allocator/internal/allocation/strategy.go
index 1f611077..83ef964b 100644
--- a/cmd/otel-allocator/internal/allocation/strategy.go
+++ b/cmd/otel-allocator/internal/allocation/strategy.go
@@ -8,8 +8,6 @@ import (
 
 	"github.com/buraksezer/consistent"
 	"github.com/go-logr/logr"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/target"
 )
@@ -22,29 +20,6 @@ var (
 		consistentHashingStrategyName: newConsistentHashingStrategy(),
 		perNodeStrategyName:           newPerNodeStrategy(),
 	}
-
-	// TargetsPerCollector records how many targets have been assigned to each collector.
-	// It is currently the responsibility of the strategy to track this information.
-	TargetsPerCollector = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "opentelemetry_allocator_targets_per_collector",
-		Help: "The number of targets for each collector.",
-	}, []string{"collector_name", "strategy"})
-	CollectorsAllocatable = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "opentelemetry_allocator_collectors_allocatable",
-		Help: "Number of collectors the allocator is able to allocate to.",
-	}, []string{"strategy"})
-	TimeToAssign = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Name: "opentelemetry_allocator_time_to_allocate",
-		Help: "The time it takes to allocate",
-	}, []string{"method", "strategy"})
-	TargetsRemaining = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "opentelemetry_allocator_targets_remaining",
-		Help: "Number of targets kept after filtering.",
-	})
-	TargetsUnassigned = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "opentelemetry_allocator_targets_unassigned",
-		Help: "Number of targets that could not be assigned due to missing node label.",
-	})
 )
 
 type Option func(Allocator)
@@ -69,13 +44,9 @@ func WithFallbackStrategy(fallbackStrategy string) Option {
 	}
 }
 
-func RecordTargetsKept(targets []*target.Item) {
-	TargetsRemaining.Set(float64(len(targets)))
-}
-
 func New(name string, log logr.Logger, opts ...Option) (Allocator, error) {
 	if strategy, ok := strategies[name]; ok {
-		return newAllocator(log.WithValues("allocator", name), strategy, opts...), nil
+		return newAllocator(log.WithValues("allocator", name), strategy, opts...)
 	}
 	return nil, fmt.Errorf("unregistered strategy: %s", name)
 }
diff --git a/cmd/otel-allocator/internal/collector/collector.go b/cmd/otel-allocator/internal/collector/collector.go
index 2f53a5e3..787dd209 100644
--- a/cmd/otel-allocator/internal/collector/collector.go
+++ b/cmd/otel-allocator/internal/collector/collector.go
@@ -4,11 +4,12 @@
 package collector
 
 import (
+	"context"
 	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/metric"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/informers"
@@ -23,19 +24,13 @@ const (
 	defaultMinUpdateInterval = time.Second * 5
 )
 
-var (
-	collectorsDiscovered = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "opentelemetry_allocator_collectors_discovered",
-		Help: "Number of collectors discovered.",
-	})
-)
-
 type Watcher struct {
 	log                          logr.Logger
 	k8sClient                    kubernetes.Interface
 	close                        chan struct{}
 	minUpdateInterval            time.Duration
 	collectorNotReadyGracePeriod time.Duration
+	collectorsDiscovered         metric.Int64Gauge
 }
 
 func NewCollectorWatcher(logger logr.Logger, kubeConfig *rest.Config, collectorNotReadyGracePeriod time.Duration) (*Watcher, error) {
@@ -44,12 +39,18 @@ func NewCollectorWatcher(logger logr.Logger, kubeConfig *rest.Config, collectorN
 		return &Watcher{}, err
 	}
 
+	meter := otel.GetMeterProvider().Meter("targetallocator")
+	collectorsDiscovered, err := meter.Int64Gauge("opentelemetry_allocator_collectors_discovered", metric.WithDescription("Number of collectors discovered."))
+	if err != nil {
+		return &Watcher{}, err
+	}
 	return &Watcher{
 		log:                          logger.WithValues("component", "opentelemetry-targetallocator"),
 		k8sClient:                    clientset,
 		close:                        make(chan struct{}),
 		minUpdateInterval:            defaultMinUpdateInterval,
 		collectorNotReadyGracePeriod: collectorNotReadyGracePeriod,
+		collectorsDiscovered:         collectorsDiscovered,
 	}, nil
 }
 
@@ -134,7 +135,7 @@ func (k *Watcher) runOnCollectors(store cache.Store, fn func(collectors map[stri
 
 		collectorMap[pod.Name] = allocation.NewCollector(pod.Name, pod.Spec.NodeName)
 	}
-	collectorsDiscovered.Set(float64(len(collectorMap)))
+	k.collectorsDiscovered.Record(context.Background(), int64(len(collectorMap)))
 	fn(collectorMap)
 }
 
diff --git a/cmd/otel-allocator/internal/server/server.go b/cmd/otel-allocator/internal/server/server.go
index 01af8f8a..d42cde84 100644
--- a/cmd/otel-allocator/internal/server/server.go
+++ b/cmd/otel-allocator/internal/server/server.go
@@ -20,25 +20,19 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/go-logr/logr"
 	"github.com/goccy/go-json"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	promcommconfig "github.com/prometheus/common/config"
 	promconfig "github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/labels"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 	"gopkg.in/yaml.v2"
 
 	"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/allocation"
 	"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/target"
 )
 
-var (
-	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Name: "opentelemetry_allocator_http_duration_seconds",
-		Help: "Duration of received HTTP requests.",
-	}, []string{"path"})
-)
-
 type collectorJSON struct {
 	Link string        `json:"_link"`
 	Jobs []*targetJSON `json:"targets"`
@@ -65,6 +59,7 @@ type Server struct {
 	mtx                                  sync.RWMutex
 	scrapeConfigResponse                 []byte
 	ScrapeConfigMarshalledSecretResponse []byte
+	httpDuration                         metric.Float64Histogram
 }
 
 type Option func(*Server)
@@ -103,10 +98,16 @@ func (s *Server) setRouter(router *gin.Engine) {
 	registerPprof(router.Group("/debug/pprof/"))
 }
 
-func NewServer(log logr.Logger, allocator allocation.Allocator, listenAddr string, options ...Option) *Server {
+func NewServer(log logr.Logger, allocator allocation.Allocator, listenAddr string, options ...Option) (*Server, error) {
+	meter := otel.GetMeterProvider().Meter("targetallocator")
+	httpDuration, err := meter.Float64Histogram("opentelemetry_allocator_http_duration_seconds", metric.WithDescription("Duration of received HTTP requests."))
+	if err != nil {
+		return nil, err
+	}
 	s := &Server{
-		logger:    log,
-		allocator: allocator,
+		logger:       log,
+		allocator:    allocator,
+		httpDuration: httpDuration,
 	}
 
 	gin.SetMode(gin.ReleaseMode)
@@ -119,7 +120,7 @@ func NewServer(log logr.Logger, allocator allocation.Allocator, listenAddr strin
 		opt(s)
 	}
 
-	return s
+	return s, nil
 }
 
 func (s *Server) Start() error {
@@ -282,9 +283,9 @@ func (s *Server) LivenessProbeHandler(c *gin.Context) {
 
 func (s *Server) PrometheusMiddleware(c *gin.Context) {
 	path := c.FullPath()
-	timer := prometheus.NewTimer(httpDuration.WithLabelValues(path))
+	begin := time.Now()
 	c.Next()
-	timer.ObserveDuration()
+	s.httpDuration.Record(context.Background(), time.Since(begin).Seconds(), metric.WithAttributes(attribute.String("path", path)))
 }
 
 // IndexHandler displays the main page of the allocator. It shows the number of jobs and targets.
diff --git a/cmd/otel-allocator/internal/target/discovery.go b/cmd/otel-allocator/internal/target/discovery.go
index c72966d0..34da6b33 100644
--- a/cmd/otel-allocator/internal/target/discovery.go
+++ b/cmd/otel-allocator/internal/target/discovery.go
@@ -4,6 +4,7 @@
 package target
 
 import (
+	"context"
 	"hash"
 	"hash/fnv"
 	"sync"
@@ -11,50 +12,35 @@ import (
 
 	"github.com/go-logr/logr"
 	go_yaml "github.com/goccy/go-yaml"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/common/model"
 	promconfig "github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap/zapcore"
 
 	allocatorWatcher "github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/watcher"
 )
 
-var (
-	targetsDiscovered = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "opentelemetry_allocator_targets",
-		Help: "Number of targets discovered.",
-	}, []string{"job_name"})
-
-	processTargetsDuration = promauto.NewHistogram(prometheus.HistogramOpts{
-		Name:    "opentelemetry_allocator_process_targets_duration_seconds",
-		Help:    "Duration of processing targets.",
-		Buckets: []float64{1, 5, 10, 30, 60, 120},
-	})
-
-	processTargetGroupsDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
-		Name:    "opentelemetry_allocator_process_target_groups_duration_seconds",
-		Help:    "Duration of processing target groups.",
-		Buckets: []float64{1, 5, 10, 30, 60, 120},
-	}, []string{"job_name"})
-)
-
 type Discoverer struct {
-	log                    logr.Logger
-	manager                *discovery.Manager
-	close                  chan struct{}
-	mtxScrape              sync.Mutex // Guards the fields below.
-	configsMap             map[allocatorWatcher.EventSource][]*promconfig.ScrapeConfig
-	hook                   discoveryHook
-	scrapeConfigsHash      hash.Hash
-	scrapeConfigsUpdater   scrapeConfigsUpdater
-	targetSets             map[string][]*targetgroup.Group
-	triggerReload          chan struct{}
-	processTargetsCallBack func(targets []*Item)
+	log                         logr.Logger
+	manager                     *discovery.Manager
+	close                       chan struct{}
+	mtxScrape                   sync.Mutex // Guards the fields below.
+	configsMap                  map[allocatorWatcher.EventSource][]*promconfig.ScrapeConfig
+	hook                        discoveryHook
+	scrapeConfigsHash           hash.Hash
+	scrapeConfigsUpdater        scrapeConfigsUpdater
+	targetSets                  map[string][]*targetgroup.Group
+	triggerReload               chan struct{}
+	processTargetsCallBack      func(targets []*Item)
+	targetsDiscovered           metric.Float64Gauge
+	processTargetsDuration      metric.Float64Histogram
+	processTargetGroupsDuration metric.Float64Histogram
 }
 
 type discoveryHook interface {
@@ -65,18 +51,36 @@ type scrapeConfigsUpdater interface {
 	UpdateScrapeConfigResponse(map[string]*promconfig.ScrapeConfig) error
 }
 
-func NewDiscoverer(log logr.Logger, manager *discovery.Manager, hook discoveryHook, scrapeConfigsUpdater scrapeConfigsUpdater, setTargets func(targets []*Item)) *Discoverer {
-	return &Discoverer{
-		log:                    log,
-		manager:                manager,
-		close:                  make(chan struct{}),
-		triggerReload:          make(chan struct{}, 1),
-		configsMap:             make(map[allocatorWatcher.EventSource][]*promconfig.ScrapeConfig),
-		hook:                   hook,
-		scrapeConfigsHash:      nil, // we want the first update to succeed even if the config is empty
-		scrapeConfigsUpdater:   scrapeConfigsUpdater,
-		processTargetsCallBack: setTargets,
+func NewDiscoverer(log logr.Logger, manager *discovery.Manager, hook discoveryHook, scrapeConfigsUpdater scrapeConfigsUpdater, setTargets func(targets []*Item)) (*Discoverer, error) {
+	meter := otel.GetMeterProvider().Meter("targetallocator")
+	targetsDiscovered, err := meter.Float64Gauge("opentelemetry_allocator_targets", metric.WithDescription("Number of targets discovered."))
+	if err != nil {
+		return nil, err
 	}
+	processTargetsDuration, err := meter.Float64Histogram("opentelemetry_allocator_process_targets_duration_seconds",
+		metric.WithDescription("Duration of processing targets."), metric.WithExplicitBucketBoundaries(1, 5, 10, 30, 60, 120))
+	if err != nil {
+		return nil, err
+	}
+	processTargetGroupsDuration, err := meter.Float64Histogram("opentelemetry_allocator_process_target_groups_duration_seconds",
+		metric.WithDescription("Duration of processing target groups."), metric.WithExplicitBucketBoundaries(1, 5, 10, 30, 60, 120))
+	if err != nil {
+		return nil, err
+	}
+	return &Discoverer{
+		log:                         log,
+		manager:                     manager,
+		close:                       make(chan struct{}),
+		triggerReload:               make(chan struct{}, 1),
+		configsMap:                  make(map[allocatorWatcher.EventSource][]*promconfig.ScrapeConfig),
+		hook:                        hook,
+		scrapeConfigsHash:           nil, // we want the first update to succeed even if the config is empty
+		scrapeConfigsUpdater:        scrapeConfigsUpdater,
+		processTargetsCallBack:      setTargets,
+		targetsDiscovered:           targetsDiscovered,
+		processTargetsDuration:      processTargetsDuration,
+		processTargetGroupsDuration: processTargetGroupsDuration,
+	}, nil
 }
 
 func (m *Discoverer) ApplyConfig(source allocatorWatcher.EventSource, scrapeConfigs []*promconfig.ScrapeConfig) error {
@@ -162,8 +166,10 @@ func (m *Discoverer) reloader() {
 func (m *Discoverer) Reload() {
 	m.mtxScrape.Lock()
 	var wg sync.WaitGroup
-	timer := prometheus.NewTimer(processTargetsDuration)
-	defer timer.ObserveDuration()
+	begin := time.Now()
+	defer func() {
+		m.processTargetsDuration.Record(context.Background(), time.Since(begin).Seconds())
+	}()
 
 	// count targets and preallocate
 	targetCount := 0
@@ -194,9 +200,11 @@ func (m *Discoverer) Reload() {
 // processTargetGroups processes the target groups and returns a map of targets.
 func (m *Discoverer) processTargetGroups(jobName string, groups []*targetgroup.Group, intoTargets []*Item) {
 	builder := labels.NewBuilder(labels.Labels{})
-	timer := prometheus.NewTimer(processTargetGroupsDuration.WithLabelValues(jobName))
 
-	defer timer.ObserveDuration()
+	begin := time.Now()
+	defer func() {
+		m.processTargetGroupsDuration.Record(context.Background(), time.Since(begin).Seconds(), metric.WithAttributes(attribute.String("job.name", jobName)))
+	}()
 	var count float64 = 0
 	index := 0
 	for _, tg := range groups {
@@ -216,7 +224,7 @@ func (m *Discoverer) processTargetGroups(jobName string, groups []*targetgroup.G
 			index++
 		}
 	}
-	targetsDiscovered.WithLabelValues(jobName).Set(count)
+	m.targetsDiscovered.Record(context.Background(), count, metric.WithAttributes(attribute.String("job.name", jobName)))
 }
 
 // Run receives and saves target set updates and triggers the scraping loops reloading.
diff --git a/cmd/otel-allocator/main.go b/cmd/otel-allocator/main.go
index 6035080f..c6ec692c 100644
--- a/cmd/otel-allocator/main.go
+++ b/cmd/otel-allocator/main.go
@@ -12,8 +12,12 @@ import (
 
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/prometheus/discovery"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	otelprom "go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/metric"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
 
@@ -27,11 +31,7 @@ import (
 )
 
 var (
-	setupLog     = ctrl.Log.WithName("setup")
-	eventsMetric = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "opentelemetry_allocator_events",
-		Help: "Number of events in the channel.",
-	}, []string{"source"})
+	setupLog = ctrl.Log.WithName("setup")
 )
 
 func main() {
@@ -42,7 +42,6 @@ func main() {
 		allocator        allocation.Allocator
 		discoveryManager *discovery.Manager
 		collectorWatcher *collector.Watcher
-		promWatcher      allocatorWatcher.Watcher
 		targetDiscoverer *target.Discoverer
 
 		discoveryCancel context.CancelFunc
@@ -52,9 +51,9 @@ func main() {
 		interrupts      = make(chan os.Signal, 1)
 		errChan         = make(chan error)
 	)
-	cfg, err := config.Load(os.Args)
-	if err != nil {
-		fmt.Printf("Failed to load config: %v", err)
+	cfg, loadErr := config.Load(os.Args)
+	if loadErr != nil {
+		fmt.Printf("Failed to load config: %v", loadErr)
 		os.Exit(1)
 	}
 	ctrl.SetLogger(cfg.RootLogger)
@@ -68,10 +67,17 @@ func main() {
 	ctx := context.Background()
 	log := ctrl.Log.WithName("allocator")
 
+	metricExporter, promErr := otelprom.New()
+	if promErr != nil {
+		panic(promErr)
+	}
+	meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(metricExporter))
+	otel.SetMeterProvider(meterProvider)
+
 	allocatorPrehook = prehook.New(cfg.FilterStrategy, log)
-	allocator, err = allocation.New(cfg.AllocationStrategy, log, allocation.WithFilter(allocatorPrehook), allocation.WithFallbackStrategy(cfg.AllocationFallbackStrategy))
-	if err != nil {
-		setupLog.Error(err, "Unable to initialize allocation strategy")
+	allocator, allocErr := allocation.New(cfg.AllocationStrategy, log, allocation.WithFilter(allocatorPrehook), allocation.WithFallbackStrategy(cfg.AllocationFallbackStrategy))
+	if allocErr != nil {
+		setupLog.Error(allocErr, "Unable to initialize allocation strategy")
 		os.Exit(1)
 	}
 
@@ -84,17 +90,23 @@ func main() {
 		}
 		httpOptions = append(httpOptions, server.WithTLSConfig(tlsConfig, cfg.HTTPS.ListenAddr))
 	}
-	srv := server.NewServer(log, allocator, cfg.ListenAddr, httpOptions...)
+	srv, serverErr := server.NewServer(log, allocator, cfg.ListenAddr, httpOptions...)
+	if serverErr != nil {
+		panic(serverErr)
+	}
 
 	discoveryCtx, discoveryCancel := context.WithCancel(ctx)
-	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(prometheus.DefaultRegisterer)
-	if err != nil {
-		setupLog.Error(err, "Unable to register metrics for Prometheus service discovery")
+	sdMetrics, discErr := discovery.CreateAndRegisterSDMetrics(prometheus.DefaultRegisterer)
+	if discErr != nil {
+		setupLog.Error(discErr, "Unable to register metrics for Prometheus service discovery")
 		os.Exit(1)
 	}
 	discoveryManager = discovery.NewManager(discoveryCtx, config.NopLogger, prometheus.DefaultRegisterer, sdMetrics)
 
-	targetDiscoverer = target.NewDiscoverer(log, discoveryManager, allocatorPrehook, srv, allocator.SetTargets)
+	targetDiscoverer, targetErr := target.NewDiscoverer(log, discoveryManager, allocatorPrehook, srv, allocator.SetTargets)
+	if targetErr != nil {
+		panic(targetErr)
+	}
 	collectorWatcher, collectorWatcherErr := collector.NewCollectorWatcher(log, cfg.ClusterConfig, cfg.CollectorNotReadyGracePeriod)
 	if collectorWatcherErr != nil {
 		setupLog.Error(collectorWatcherErr, "Unable to initialize collector watcher")
@@ -104,20 +116,20 @@ func main() {
 	defer close(interrupts)
 
 	if cfg.PrometheusCR.Enabled {
-		promWatcher, err = allocatorWatcher.NewPrometheusCRWatcher(ctx, setupLog.WithName("prometheus-cr-watcher"), *cfg)
-		if err != nil {
-			setupLog.Error(err, "Can't start the prometheus watcher")
+		promWatcher, allocErr := allocatorWatcher.NewPrometheusCRWatcher(ctx, setupLog.WithName("prometheus-cr-watcher"), *cfg)
+		if allocErr != nil {
+			setupLog.Error(allocErr, "Can't start the prometheus watcher")
 			os.Exit(1)
 		}
 		// apply the initial configuration
 		promConfig, loadErr := promWatcher.LoadConfig(ctx)
 		if loadErr != nil {
-			setupLog.Error(err, "Can't load initial Prometheus configuration from Prometheus CRs")
+			setupLog.Error(loadErr, "Can't load initial Prometheus configuration from Prometheus CRs")
 			os.Exit(1)
 		}
 		loadErr = targetDiscoverer.ApplyConfig(allocatorWatcher.EventSourcePrometheusCR, promConfig.ScrapeConfigs)
 		if loadErr != nil {
-			setupLog.Error(err, "Can't load initial scrape targets from Prometheus CRs")
+			setupLog.Error(loadErr, "Can't load initial scrape targets from Prometheus CRs")
 			os.Exit(1)
 		}
 		runGroup.Add(
@@ -148,7 +160,7 @@ func main() {
 		func() error {
 			// Initial loading of the config file's scrape config
 			if cfg.PromConfig != nil && len(cfg.PromConfig.ScrapeConfigs) > 0 {
-				err = targetDiscoverer.ApplyConfig(allocatorWatcher.EventSourceConfigMap, cfg.PromConfig.ScrapeConfigs)
+				err := targetDiscoverer.ApplyConfig(allocatorWatcher.EventSourceConfigMap, cfg.PromConfig.ScrapeConfigs)
 				if err != nil {
 					setupLog.Error(err, "Unable to apply initial configuration")
 					return err
@@ -201,12 +213,17 @@ func main() {
 				}
 			})
 	}
+	meter := otel.GetMeterProvider().Meter("targetallocator")
+	eventsMetric, err := meter.Int64Counter("opentelemetry_allocator_events", metric.WithDescription("Number of events in the channel."))
+	if err != nil {
+		panic(err)
+	}
 	runGroup.Add(
 		func() error {
 			for {
 				select {
 				case event := <-eventChan:
-					eventsMetric.WithLabelValues(event.Source.String()).Inc()
+					eventsMetric.Add(context.Background(), 1, metric.WithAttributes(attribute.String("source", event.Source.String())))
 					loadConfig, err := event.Watcher.LoadConfig(ctx)
 					if err != nil {
 						setupLog.Error(err, "Unable to load configuration")



Test patch:
diff --git a/cmd/otel-allocator/benchmark_test.go b/cmd/otel-allocator/benchmark_test.go
index f39f60df..0b1b21f6 100644
--- a/cmd/otel-allocator/benchmark_test.go
+++ b/cmd/otel-allocator/benchmark_test.go
@@ -6,7 +6,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"os"
 	"strconv"
 	"strings"
 	"testing"
@@ -41,7 +40,8 @@ func BenchmarkProcessTargets(b *testing.B) {
 		tsets := prepareBenchmarkData(numTargets, targetsPerGroup, groupsPerJob)
 		for _, strategy := range allocation.GetRegisteredAllocatorNames() {
 			b.Run(fmt.Sprintf("%s/%d", strategy, numTargets), func(b *testing.B) {
-				targetDiscoverer := createTestDiscoverer(strategy, map[string][]*relabel.Config{})
+				targetDiscoverer, err := createTestDiscoverer(strategy, map[string][]*relabel.Config{})
+				require.NoError(b, err)
 				targetDiscoverer.UpdateTsets(tsets)
 				b.ResetTimer()
 				for i := 0; i < b.N; i++ {
@@ -82,7 +82,8 @@ func BenchmarkProcessTargetsWithRelabelConfig(b *testing.B) {
 
 		for _, strategy := range allocation.GetRegisteredAllocatorNames() {
 			b.Run(fmt.Sprintf("%s/%d", strategy, numTargets), func(b *testing.B) {
-				targetDiscoverer := createTestDiscoverer(strategy, prehookConfig)
+				targetDiscoverer, err := createTestDiscoverer(strategy, prehookConfig)
+				require.NoError(b, err)
 				targetDiscoverer.UpdateTsets(tsets)
 				b.ResetTimer()
 				for i := 0; i < b.N; i++ {
@@ -167,21 +168,26 @@ func prepareBenchmarkData(numTargets, targetsPerGroup, groupsPerJob int) map[str
 	return tsets
 }
 
-func createTestDiscoverer(allocationStrategy string, prehookConfig map[string][]*relabel.Config) *target.Discoverer {
+func createTestDiscoverer(allocationStrategy string, prehookConfig map[string][]*relabel.Config) (*target.Discoverer, error) {
 	ctx := context.Background()
 	logger := ctrl.Log.WithName(fmt.Sprintf("bench-%s", allocationStrategy))
 	ctrl.SetLogger(logr.New(log.NullLogSink{}))
 	allocatorPrehook := prehook.New("relabel-config", logger)
 	allocatorPrehook.SetConfig(prehookConfig)
 	allocator, err := allocation.New(allocationStrategy, logger, allocation.WithFilter(allocatorPrehook))
-	srv := server.NewServer(logger, allocator, "localhost:0")
 	if err != nil {
-		setupLog.Error(err, "Unable to initialize allocation strategy")
-		os.Exit(1)
+		return nil, err
+	}
+	srv, err := server.NewServer(logger, allocator, "localhost:0")
+	if err != nil {
+		return nil, err
 	}
 	registry := prometheus.NewRegistry()
 	sdMetrics, _ := discovery.CreateAndRegisterSDMetrics(registry)
 	discoveryManager := discovery.NewManager(ctx, config.NopLogger, registry, sdMetrics)
-	targetDiscoverer := target.NewDiscoverer(logger, discoveryManager, allocatorPrehook, srv, allocator.SetTargets)
-	return targetDiscoverer
+	targetDiscoverer, err := target.NewDiscoverer(logger, discoveryManager, allocatorPrehook, srv, allocator.SetTargets)
+	if err != nil {
+		return nil, err
+	}
+	return targetDiscoverer, nil
 }
diff --git a/cmd/otel-allocator/internal/collector/collector_test.go b/cmd/otel-allocator/internal/collector/collector_test.go
index 17dc2844..9442084b 100644
--- a/cmd/otel-allocator/internal/collector/collector_test.go
+++ b/cmd/otel-allocator/internal/collector/collector_test.go
@@ -9,9 +9,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/embedded"
+	"go.uber.org/atomic"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
@@ -29,6 +31,15 @@ var labelSelector = metav1.LabelSelector{
 	MatchLabels: labelMap,
 }
 
+type reportingGauge struct {
+	embedded.Int64Gauge
+	value atomic.Int64
+}
+
+func (r *reportingGauge) Record(_ context.Context, value int64, _ ...metric.RecordOption) {
+	r.value.Store(value)
+}
+
 func getTestPodWatcher(collectorNotReadyGracePeriod time.Duration) Watcher {
 	podWatcher := Watcher{
 		k8sClient:                    fake.NewSimpleClientset(),
@@ -36,6 +47,7 @@ func getTestPodWatcher(collectorNotReadyGracePeriod time.Duration) Watcher {
 		log:                          logger,
 		minUpdateInterval:            time.Millisecond,
 		collectorNotReadyGracePeriod: collectorNotReadyGracePeriod,
+		collectorsDiscovered:         &reportingGauge{},
 	}
 	return podWatcher
 }
@@ -191,7 +203,7 @@ func Test_runWatch(t *testing.T) {
 				defer mapMutex.Unlock()
 				assert.Len(collect, actual, len(tt.want))
 				assert.Equal(collect, tt.want, actual)
-				assert.Equal(collect, testutil.ToFloat64(collectorsDiscovered), float64(len(actual)))
+				assert.Equal(collect, podWatcher.collectorsDiscovered.(*reportingGauge).value.Load(), int64(len(actual)))
 			}, time.Second*30, time.Millisecond*100)
 		})
 	}
@@ -313,7 +325,7 @@ func Test_gracePeriodWithNonRunningPodPhase(t *testing.T) {
 				defer mapMutex.Unlock()
 				assert.Len(collect, actual, len(tt.want))
 				assert.Equal(collect, actual, tt.want)
-				assert.Equal(collect, testutil.ToFloat64(collectorsDiscovered), float64(len(actual)))
+				assert.Equal(collect, podWatcher.collectorsDiscovered.(*reportingGauge).value.Load(), int64(len(actual)))
 			}, time.Second*3, time.Millisecond)
 		})
 	}
@@ -436,7 +448,7 @@ func Test_gracePeriodWithNonReadyPodCondition(t *testing.T) {
 				defer mapMutex.Unlock()
 				assert.Len(collect, actual, len(tt.want))
 				assert.Equal(collect, actual, tt.want)
-				assert.Equal(collect, testutil.ToFloat64(collectorsDiscovered), float64(len(actual)))
+				assert.Equal(collect, podWatcher.collectorsDiscovered.(*reportingGauge).value.Load(), int64(len(actual)))
 			}, time.Second*3, time.Millisecond)
 		})
 	}
diff --git a/cmd/otel-allocator/internal/server/bench_test.go b/cmd/otel-allocator/internal/server/bench_test.go
index 98b88c0f..7033c1fb 100644
--- a/cmd/otel-allocator/internal/server/bench_test.go
+++ b/cmd/otel-allocator/internal/server/bench_test.go
@@ -15,6 +15,7 @@ import (
 	promconfig "github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/allocation"
 	"github.com/open-telemetry/opentelemetry-operator/cmd/otel-allocator/internal/target"
@@ -44,7 +45,8 @@ func BenchmarkServerTargetsHandler(b *testing.B) {
 			listenAddr := ":8080"
 			a.SetCollectors(cols)
 			a.SetTargets(targets)
-			s := NewServer(logger, a, listenAddr)
+			s, err := NewServer(logger, a, listenAddr)
+			require.NoError(b, err)
 			b.Run(fmt.Sprintf("%s_num_cols_%d_num_jobs_%d", allocatorName, v.numCollectors, v.numJobs), func(b *testing.B) {
 				b.ReportAllocs()
 				for i := 0; i < b.N; i++ {
diff --git a/cmd/otel-allocator/internal/server/server_test.go b/cmd/otel-allocator/internal/server/server_test.go
index c5329c64..ac7adff7 100644
--- a/cmd/otel-allocator/internal/server/server_test.go
+++ b/cmd/otel-allocator/internal/server/server_test.go
@@ -47,7 +47,8 @@ var (
 func TestServer_LivenessProbeHandler(t *testing.T) {
 	leastWeighted, _ := allocation.New("least-weighted", logger)
 	listenAddr := ":8080"
-	s := NewServer(logger, leastWeighted, listenAddr)
+	s, err := NewServer(logger, leastWeighted, listenAddr)
+	require.NoError(t, err)
 	request := httptest.NewRequest("GET", "/livez", nil)
 	w := httptest.NewRecorder()
 
@@ -161,7 +162,8 @@ func TestServer_TargetsHandler(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, tt.args.allocator, listenAddr)
+			s, err := NewServer(logger, tt.args.allocator, listenAddr)
+			require.NoError(t, err)
 
 			tt.args.allocator.SetCollectors(map[string]*allocation.Collector{"test-collector": {Name: "test-collector"}})
 			tt.args.allocator.SetTargets(tt.args.targets)
@@ -501,7 +503,8 @@ func TestServer_ScrapeConfigsHandler(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, nil, listenAddr, tc.serverOptions...)
+			s, err := NewServer(logger, nil, listenAddr, tc.serverOptions...)
+			require.NoError(t, err)
 			assert.NoError(t, s.UpdateScrapeConfigResponse(tc.scrapeConfigs))
 
 			request := httptest.NewRequest("GET", "/scrape_configs", nil)
@@ -592,7 +595,8 @@ func TestServer_JobHandler(t *testing.T) {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
 			a := &mockAllocator{targetItems: tc.targetItems}
-			s := NewServer(logger, a, listenAddr)
+			s, err := NewServer(logger, a, listenAddr)
+			require.NoError(t, err)
 			request := httptest.NewRequest("GET", "/jobs", nil)
 			w := httptest.NewRecorder()
 
@@ -652,7 +656,8 @@ func TestServer_JobsHandler_HTML(t *testing.T) {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
 			a := &mockAllocator{targetItems: tc.targetItems}
-			s := NewServer(logger, a, listenAddr)
+			s, err := NewServer(logger, a, listenAddr)
+			require.NoError(t, err)
 			a.SetCollectors(map[string]*allocation.Collector{
 				"test-collector":  {Name: "test-collector"},
 				"test-collector2": {Name: "test-collector2"},
@@ -721,7 +726,8 @@ func TestServer_JobHandler_HTML(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, tt.args.allocator, listenAddr)
+			s, err := NewServer(logger, tt.args.allocator, listenAddr)
+			require.NoError(t, err)
 			tt.args.allocator.SetCollectors(map[string]*allocation.Collector{
 				"test-collector":  {Name: "test-collector"},
 				"test-collector2": {Name: "test-collector2"},
@@ -779,7 +785,8 @@ func TestServer_IndexHandler(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, tc.allocator, listenAddr)
+			s, err := NewServer(logger, tc.allocator, listenAddr)
+			require.NoError(t, err)
 			tc.allocator.SetCollectors(map[string]*allocation.Collector{
 				"test-collector1": {Name: "test-collector1"},
 				"test-collector2": {Name: "test-collector2"},
@@ -836,7 +843,8 @@ func TestServer_TargetsHTMLHandler(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, tc.allocator, listenAddr)
+			s, err := NewServer(logger, tc.allocator, listenAddr)
+			require.NoError(t, err)
 			tc.allocator.SetCollectors(map[string]*allocation.Collector{
 				"test-collector1": {Name: "test-collector1"},
 				"test-collector2": {Name: "test-collector2"},
@@ -923,7 +931,8 @@ func TestServer_CollectorHandler(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, tc.allocator, listenAddr)
+			s, err := NewServer(logger, tc.allocator, listenAddr)
+			require.NoError(t, err)
 			tc.allocator.SetCollectors(map[string]*allocation.Collector{
 				"test-collector":  {Name: "test-collector"},
 				"test-collector2": {Name: "test-collector2"},
@@ -989,7 +998,8 @@ func TestServer_TargetHTMLHandler(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, tc.allocator, listenAddr)
+			s, err := NewServer(logger, tc.allocator, listenAddr)
+			require.NoError(t, err)
 			tc.allocator.SetCollectors(map[string]*allocation.Collector{
 				"test-collector":  {Name: "test-collector"},
 				"test-collector2": {Name: "test-collector2"},
@@ -1060,7 +1070,8 @@ func TestServer_Readiness(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, nil, listenAddr)
+			s, err := NewServer(logger, nil, listenAddr)
+			require.NoError(t, err)
 			if tc.scrapeConfigs != nil {
 				assert.NoError(t, s.UpdateScrapeConfigResponse(tc.scrapeConfigs))
 			}
@@ -1101,10 +1112,11 @@ func TestServer_ScrapeConfigResponse(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			listenAddr := ":8080"
-			s := NewServer(logger, nil, listenAddr)
+			s, err := NewServer(logger, nil, listenAddr)
+			require.NoError(t, err)
 
 			allocCfg := allocatorconfig.CreateDefaultConfig()
-			err := allocatorconfig.LoadFromFile(tc.filePath, &allocCfg)
+			err = allocatorconfig.LoadFromFile(tc.filePath, &allocCfg)
 			require.NoError(t, err)
 
 			jobToScrapeConfig := make(map[string]*promconfig.ScrapeConfig)
diff --git a/cmd/otel-allocator/internal/target/discovery_test.go b/cmd/otel-allocator/internal/target/discovery_test.go
index a3d907fa..23e21b25 100644
--- a/cmd/otel-allocator/internal/target/discovery_test.go
+++ b/cmd/otel-allocator/internal/target/discovery_test.go
@@ -63,13 +63,14 @@ func TestDiscovery(t *testing.T) {
 	require.NoError(t, err)
 	d := discovery.NewManager(ctx, config.NopLogger, registry, sdMetrics)
 	results := make(chan []string)
-	manager := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, func(targets []*Item) {
+	manager, err := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, func(targets []*Item) {
 		var result []string
 		for _, t := range targets {
 			result = append(result, t.TargetURL)
 		}
 		results <- result
 	})
+	require.NoError(t, err)
 
 	defer func() { manager.Close() }()
 	defer cancelFunc()
@@ -310,7 +311,8 @@ func TestDiscovery_ScrapeConfigHashing(t *testing.T) {
 	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(registry)
 	require.NoError(t, err)
 	d := discovery.NewManager(ctx, config.NopLogger, registry, sdMetrics)
-	manager := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, nil)
+	manager, err := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, nil)
+	require.NoError(t, err)
 
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
@@ -401,11 +403,12 @@ func TestDiscoveryTargetHashing(t *testing.T) {
 	require.NoError(t, err)
 	d := discovery.NewManager(ctx, config.NopLogger, registry, sdMetrics)
 	results := make(chan []*Item)
-	manager := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, func(targets []*Item) {
+	manager, err := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, func(targets []*Item) {
 		var result []*Item
 		result = append(result, targets...)
 		results <- result
 	})
+	require.NoError(t, err)
 
 	defer manager.Close()
 	defer cancelFunc()
@@ -449,7 +452,8 @@ func TestDiscovery_NoConfig(t *testing.T) {
 	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(registry)
 	require.NoError(t, err)
 	d := discovery.NewManager(ctx, config.NopLogger, registry, sdMetrics)
-	manager := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, nil)
+	manager, err := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, nil)
+	require.NoError(t, err)
 	defer close(manager.close)
 	defer cancelFunc()
 
@@ -487,7 +491,14 @@ func TestProcessTargetGroups_StableLabelIterationOrder(t *testing.T) {
 	}
 
 	results := make([]*Item, 1)
-	var d *Discoverer
+	scu := &mockScrapeConfigUpdater{}
+	ctx := context.Background()
+	registry := prometheus.NewRegistry()
+	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(registry)
+	require.NoError(t, err)
+	manager := discovery.NewManager(ctx, config.NopLogger, registry, sdMetrics)
+	d, err := NewDiscoverer(ctrl.Log.WithName("test"), manager, nil, scu, nil)
+	require.NoError(t, err)
 	d.processTargetGroups("test", groups, results)
 
 	for i, l := range results[0].Labels {
@@ -534,7 +545,8 @@ func BenchmarkApplyScrapeConfig(b *testing.B) {
 	sdMetrics, err := discovery.CreateAndRegisterSDMetrics(registry)
 	require.NoError(b, err)
 	d := discovery.NewManager(ctx, config.NopLogger, registry, sdMetrics)
-	manager := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, nil)
+	manager, err := NewDiscoverer(ctrl.Log.WithName("test"), d, nil, scu, nil)
+	require.NoError(b, err)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
diff --git a/internal/controllers/reconcile_test.go b/internal/controllers/reconcile_test.go
index 19b48f9f..3c825b46 100644
--- a/internal/controllers/reconcile_test.go
+++ b/internal/controllers/reconcile_test.go
@@ -1491,6 +1491,7 @@ func getAllOwnedResources(
 ) ([]client.Object, error) {
 	ownedResourceTypes := reconciler.GetOwnedResourceTypes()
 	allResources := []client.Object{}
+	copyOfOwner := *owner
 	for _, resourceType := range ownedResourceTypes {
 		list := &unstructured.UnstructuredList{}
 		gvk, err := apiutil.GVKForObject(resourceType, k8sClient.Scheme())
@@ -1508,7 +1509,7 @@ func getAllOwnedResources(
 			}
 
 			newObj := obj
-			if !IsOwnedBy(&newObj, owner) {
+			if !IsOwnedBy(&newObj, &copyOfOwner) {
 				continue
 			}
 			allResources = append(allResources, &newObj)