Repository: caddyserver/caddy (language: go)

Issue:
panic: duplicate metrics collector registration attempted
Created latest development with xcaddy v0.4.4 on Windows 11 x64

`xcaddy build master --with github.com/caddy-dns/cloudflare --with github.com/caddyserver/transform-encoder --with github.com/WeidiDeng/caddy-cloudflare-ip --with github.com/porech/caddy-maxmind-geolocation --with github.com/mholt/caddy-webdav`


caddy run


```
2025/01/28 07:45:17.438 INFO    maxprocs: Leaving GOMAXPROCS=12: CPU quota undefined
2025/01/28 07:45:17.438 INFO    GOMEMLIMIT is updated   {"package": "github.com/KimMachineGun/automemlimit/memlimit", "GOMEMLIMIT": 61564166553, "previous": 9223372036854775807}
2025/01/28 07:45:17.452 INFO    using adjacent Caddyfile
2025/01/28 07:45:17.464 INFO    adapted config to JSON  {"adapter": "caddyfile"}
2025/01/28 07:45:17.464 WARN    Caddyfile input is not formatted; run 'caddy fmt --overwrite' to fix inconsistencies   {"adapter": "caddyfile", "file": "Caddyfile", "line": 201}
2025/01/28 07:45:17.480 INFO    admin   admin endpoint started  {"address": "localhost:2019", "enforce_origin": false, "origins": ["//127.0.0.1:2019", "//localhost:2019", "//[::1]:2019"]}
2025/01/28 07:45:17.480 INFO    tls.cache.maintenance   started background certificate maintenance      {"cache": "0xc0005ea380"}
2025/01/28 07:45:17.481 INFO    http.auto_https server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS {"server_name": "srv0", "https_port": 443}
2025/01/28 07:45:17.481 INFO    http.auto_https automatic HTTP->HTTPS redirects are disabled    {"server_name": "srv0"}
2025/01/28 07:45:17.485 INFO    maxprocs: No GOMAXPROCS change to reset
panic: duplicate metrics collector registration attempted

goroutine 1 [running]:
github.com/prometheus/client_golang/prometheus.(*Registry).MustRegister(0xc0005f0a00, {0xc0007fa610?, 0x199dcb6?, 0xd?})        github.com/prometheus/client_golang@v1.19.1/prometheus/registry.go:405 +0x66
github.com/prometheus/client_golang/prometheus/promauto.Factory.NewGaugeVec({{0x1e4c9f0?, 0xc0005f0a00?}}, {{0x1988a69, 0x5}, {0x199dcb6, 0xd}, {0x19a7e20, 0x11}, {0x19e3598, 0x29}, ...}, ...)
        github.com/prometheus/client_golang@v1.19.1/prometheus/promauto/auto.go:308 +0x163
github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.initReverseProxyMetrics(0xc0007fc000, 0xc0005f0a00)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/reverseproxy/metrics.go:24 +0x7c
github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.newMetricsUpstreamsHealthyUpdater(0xc0007fc000, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x4, 0x4}, {0x0, 0x0, ...}, ...})
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/reverseproxy/metrics.go:42 +0x65
github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).Provision(0xc0007fc000, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x4, 0x4}, {0x0, 0x0, ...}, ...})
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/reverseproxy/reverseproxy.go:385 +0x12b8
github.com/caddyserver/caddy/v2.Context.LoadModuleByID({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x4, 0x4}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:391 +0x695
github.com/caddyserver/caddy/v2.Context.loadModuleInline({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x3, 0x4}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:441 +0xf0
github.com/caddyserver/caddy/v2.Context.LoadModule({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x3, 0x4}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:222 +0x706
github.com/caddyserver/caddy/v2/modules/caddyhttp.(*Route).ProvisionHandlers(0xc0004457a8, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x3, 0x4}, {0x0, 0x0, ...}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/routes.go:154 +0x71
github.com/caddyserver/caddy/v2/modules/caddyhttp.RouteList.ProvisionHandlers({0xc000445688, 0x3, 0x29495bc2ea8?}, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x3, 0x4}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/routes.go:212 +0xa9
github.com/caddyserver/caddy/v2/modules/caddyhttp.RouteList.Provision({0xc000445688, 0x3, 0x4}, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x3, 0x4}, ...})
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/routes.go:189 +0xb2
github.com/caddyserver/caddy/v2/modules/caddyhttp.(*Subroute).Provision(0xc0000833a0, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x3, 0x4}, {0x0, 0x0, ...}, ...})
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/subroute.go:58 +0x5f
github.com/caddyserver/caddy/v2.Context.LoadModuleByID({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0002d06c0, 0x3, 0x4}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:391 +0x695
github.com/caddyserver/caddy/v2.Context.loadModuleInline({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc000082ea0, 0x2, 0x2}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:441 +0xf0
github.com/caddyserver/caddy/v2.Context.LoadModule({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc000082ea0, 0x2, 0x2}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:222 +0x706
github.com/caddyserver/caddy/v2/modules/caddyhttp.(*Route).ProvisionHandlers(0xc0004455b8, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc000082ea0, 0x2, 0x2}, {0x0, 0x0, ...}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/routes.go:154 +0x71
github.com/caddyserver/caddy/v2/modules/caddyhttp.RouteList.ProvisionHandlers({0xc000445408, 0x4, 0x29495bc2ea8?}, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc000082ea0, 0x2, 0x2}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/routes.go:212 +0xa9
github.com/caddyserver/caddy/v2/modules/caddyhttp.RouteList.Provision({0xc000445408, 0x4, 0x4}, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc000082ea0, 0x2, 0x2}, ...})
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/routes.go:189 +0xb2
github.com/caddyserver/caddy/v2/modules/caddyhttp.(*Subroute).Provision(0xc000082e60, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc000082ea0, 0x2, 0x2}, {0x0, 0x0, ...}, ...})
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/subroute.go:58 +0x5f
github.com/caddyserver/caddy/v2.Context.LoadModuleByID({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc000082ea0, 0x2, 0x2}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:391 +0x695
github.com/caddyserver/caddy/v2.Context.loadModuleInline({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0009be400, 0x1, 0x1}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:441 +0xf0
github.com/caddyserver/caddy/v2.Context.LoadModule({{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0009be400, 0x1, 0x1}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:222 +0x706
github.com/caddyserver/caddy/v2/modules/caddyhttp.(*Route).ProvisionHandlers(0xc0002115b8, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0009be400, 0x1, 0x1}, {0x0, 0x0, ...}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/routes.go:154 +0x71
github.com/caddyserver/caddy/v2/modules/caddyhttp.RouteList.ProvisionHandlers({0xc000211408, 0x6, 0x19a78e1?}, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0009be400, 0x1, 0x1}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/routes.go:212 +0xa9
github.com/caddyserver/caddy/v2/modules/caddyhttp.(*App).Provision(0xc000651110, {{0x1e4f7a8, 0xc00016ecf0}, 0xc0005eecc0, 0xc0001551a0, {0xc0009be400, 0x1, 0x1}, {0x0, 0x0, ...}, ...})
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/modules/caddyhttp/app.go:367 +0xf25
github.com/caddyserver/caddy/v2.Context.LoadModuleByID({{0x1e4f7e0, 0xc0005f0a50}, 0xc0005eecc0, 0xc0001551a0, {0xc0009be400, 0x1, 0x1}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:391 +0x695
github.com/caddyserver/caddy/v2.Context.App({{0x1e4f7e0, 0xc0005f0a50}, 0xc0005eecc0, 0xc0001551a0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, ...}, ...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/context.go:467 +0xee
github.com/caddyserver/caddy/v2.provisionContext.func3(...)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/caddy.go:545
github.com/caddyserver/caddy/v2.provisionContext(0xc0005fb3e8?, 0x1)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/caddy.go:550 +0x53c
github.com/caddyserver/caddy/v2.run(0xc?, 0x1)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/caddy.go:400 +0x78
github.com/caddyserver/caddy/v2.unsyncedDecodeAndRun({0xc0005a8a80, 0x6714, 0x6a80}, 0x1)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/caddy.go:343 +0x145
github.com/caddyserver/caddy/v2.changeConfig({0x19870e5, 0x4}, {0x198df60, 0x7}, {0xc0005a2000, 0x6714, 0x6a80}, {0x0, 0x0}, 0x1)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/caddy.go:234 +0x6b6
github.com/caddyserver/caddy/v2.Load({0xc0005a2000?, 0x0?, 0x0?}, 0x0?)
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/caddy.go:133 +0x114
github.com/caddyserver/caddy/v2/cmd.cmdRun({0x0?})
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/cmd/commandfuncs.go:235 +0x831
github.com/caddyserver/caddy/v2/cmd.init.1.func2.WrapCommandFuncForCobra.1(0xc0000e4908, {0x1987145?, 0x4?, 0x1987115?})        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/cmd/cobra.go:141 +0x2f
github.com/spf13/cobra.(*Command).execute(0xc0000e4908, {0x2b441e0, 0x0, 0x0})
        github.com/spf13/cobra@v1.8.1/command.go:985 +0xaaa
github.com/spf13/cobra.(*Command).ExecuteC(0xc0000e4308)
        github.com/spf13/cobra@v1.8.1/command.go:1117 +0x3ff
github.com/spf13/cobra.(*Command).Execute(...)
        github.com/spf13/cobra@v1.8.1/command.go:1041
github.com/caddyserver/caddy/v2/cmd.Main()
        github.com/caddyserver/caddy/v2@v2.9.2-0.20250127213054-904a0fa368b7/cmd/main.go:96 +0x46b
main.main()
        caddy/main.go:16 +0xf
```


Discussion / hints:
Sigh, I guess #6806 wasn't perfect. Can you share how to reproduce it? Because I did try multiple reloads and change of upstreams in my tests. It helps me to know what I missed.

Oh, and thanks for trying the bleeding edge!
> Sigh, I guess [#6806](https://github.com/caddyserver/caddy/pull/6806) wasn't perfect. Can you share how to reproduce it? Because I did try multiple reloads and change of upstreams in my tests. It helps me to know what I missed.
> 
> Oh, and thanks for trying the bleeding edge!

I just narrowed it down to simple caddyfile having one reverse_proxy in  in each and it will fail

```
192.168.1.2 {
	reverse_proxy 192.168.1.5 {
	}
}

192.168.1.3 {
	reverse_proxy 192.168.1.6 {
	}
}
```
same problam

if use 2 `reverse_proxy`, it will break&panic

example-Caddyfile

```caddy
1.example.xyz {
	reverse_proxy * 127.0.0.1:80
}

2.example.xyz {
	reverse_proxy * 127.0.0.1:8080
}
```

build info : v2.9.2-0.20250127213054-904a0fa368b7
Ran into the same issue. Can confirm these configuration files make caddy crash on boot.
Having the same issue on the latest builds from master.

Fix patch:
diff --git a/modules/caddyhttp/reverseproxy/metrics.go b/modules/caddyhttp/reverseproxy/metrics.go
index 8a7105c1193..2488427304e 100644
--- a/modules/caddyhttp/reverseproxy/metrics.go
+++ b/modules/caddyhttp/reverseproxy/metrics.go
@@ -1,11 +1,12 @@
 package reverseproxy
 
 import (
+	"errors"
 	"runtime/debug"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
@@ -13,6 +14,7 @@ import (
 )
 
 var reverseProxyMetrics = struct {
+	once             sync.Once
 	upstreamsHealthy *prometheus.GaugeVec
 	logger           *zap.Logger
 }{}
@@ -21,12 +23,25 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) {
 	const ns, sub = "caddy", "reverse_proxy"
 
 	upstreamsLabels := []string{"upstream"}
-	reverseProxyMetrics.upstreamsHealthy = promauto.With(registry).NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: ns,
-		Subsystem: sub,
-		Name:      "upstreams_healthy",
-		Help:      "Health status of reverse proxy upstreams.",
-	}, upstreamsLabels)
+	reverseProxyMetrics.once.Do(func() {
+		reverseProxyMetrics.upstreamsHealthy = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: ns,
+			Subsystem: sub,
+			Name:      "upstreams_healthy",
+			Help:      "Health status of reverse proxy upstreams.",
+		}, upstreamsLabels)
+	})
+
+	// duplicate registration could happen if multiple sites with reverse proxy are configured; so ignore the error because
+	// there's no good way to capture having multiple sites with reverse proxy. If this happens, the metrics will be
+	// registered twice, but the second registration will be ignored.
+	if err := registry.Register(reverseProxyMetrics.upstreamsHealthy); err != nil &&
+		!errors.Is(err, prometheus.AlreadyRegisteredError{
+			ExistingCollector: reverseProxyMetrics.upstreamsHealthy,
+			NewCollector:      reverseProxyMetrics.upstreamsHealthy,
+		}) {
+		panic(err)
+	}
 
 	reverseProxyMetrics.logger = handler.logger.Named("reverse_proxy.metrics")
 }
@@ -35,13 +50,8 @@ type metricsUpstreamsHealthyUpdater struct {
 	handler *Handler
 }
 
-const upstreamsHealthyMetrics caddy.CtxKey = "reverse_proxy_upstreams_healthy"
-
 func newMetricsUpstreamsHealthyUpdater(handler *Handler, ctx caddy.Context) *metricsUpstreamsHealthyUpdater {
-	if set := ctx.Value(upstreamsHealthyMetrics); set == nil {
-		initReverseProxyMetrics(handler, ctx.GetMetricsRegistry())
-		ctx = ctx.WithValue(upstreamsHealthyMetrics, true)
-	}
+	initReverseProxyMetrics(handler, ctx.GetMetricsRegistry())
 	reverseProxyMetrics.upstreamsHealthy.Reset()
 
 	return &metricsUpstreamsHealthyUpdater{handler}


Test patch: