Repository: caddyserver/caddy (language: go)

Issue:
reverseproxy dynamic upstream race
Running off of master (d8135505d38ca872d507418d2017f4dbf352f06a)
I am running into write-write race conditions with:

` 
| WARNING: DATA RACE
  | Write at 0x00c00015d768 by goroutine 1181:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1072 +0x79
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21b7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |  /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
`

`  
  | WARNING: DATA RACE
  | Write at 0x00c000798820 by goroutine 852:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
`


Discussion / hints:
`
  | WARNING: DATA RACE
  | Read at 0x00c0005abe60 by goroutine 654:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Healthy()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:83 +0x6e
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Available()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:75 +0x44
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.selectRandomHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:750 +0xe4
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.RandomSelection.Select()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:66 +0x66
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*RandomSelection).Select()
  |       <autogenerated>:1 +0x106
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:478 +0x5c1
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
`

  `
  | Previous write at 0x00c0005abe60 by goroutine 655:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1092 +0x2fb
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21b7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
  `
`
  | Previous read at 0x00c0005abe68 by goroutine 659:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).reverseProxy()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:809 +0x15f1
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:530 +0x173d
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
`
Did you customize Caddy in any way? The referenced line 1072 in `github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1072` is an empty line. Please share the information listed below.

<pre><code>## 1. Environment

### 1a. Operating system and version

```
paste here
```


### 1b. Caddy version (run `caddy version` or paste commit SHA)

```
paste here
```


### 1c. Go version (if building Caddy from source; run `go version`)

```
paste here
```


## 2. Reproducer

### 2.1 Caddy configuration (Caddyfile or JSON)

```
paste here
```

</code></pre>
oh my bad
yes i tried to fix this race condition, and it just exposed another one.
the point is these methods are not threadsafe
its part of a larger config, but the important part is its a simple reverse proxy with a AUpstreams with an A record
i am in the airport can post it later


```
diff --git a/modules/caddyhttp/reverseproxy/healthchecks.go b/modules/caddyhttp/reverseproxy/healthchecks.go
index 80b635ad..a40d1d9d 100644
--- a/modules/caddyhttp/reverseproxy/healthchecks.go
+++ b/modules/caddyhttp/reverseproxy/healthchecks.go
@@ -455,8 +455,11 @@ func (h *Handler) countFailure(upstream *Upstream) {
 		return
 	}
 
+	// load host
+	host := upstream.Host()
+
 	// count failure immediately
-	err := upstream.Host.countFail(1)
+	err := host.countFail(1)
 	if err != nil {
 		h.HealthChecks.Passive.logger.Error("could not count failure",
 			zap.String("host", upstream.Dial),
@@ -487,5 +490,5 @@ func (h *Handler) countFailure(upstream *Upstream) {
 				zap.String("host", upstream.Dial),
 				zap.Error(err))
 		}
-	}(upstream.Host, failDuration)
+	}(host, failDuration)
 }
diff --git a/modules/caddyhttp/reverseproxy/hosts.go b/modules/caddyhttp/reverseproxy/hosts.go
index a23715ec..62a7475e 100644
--- a/modules/caddyhttp/reverseproxy/hosts.go
+++ b/modules/caddyhttp/reverseproxy/hosts.go
@@ -33,8 +33,6 @@ type UpstreamPool []*Upstream
 // state of the backend host it is correlated with.
 // Upstream values must not be copied.
 type Upstream struct {
-	*Host `json:"-"`
-
 	// The [network address](/docs/conventions#network-addresses)
 	// to dial to connect to the upstream. Must represent precisely
 	// one socket (i.e. no port ranges). A valid network address
@@ -83,7 +81,7 @@ func (u *Upstream) Available() bool {
 func (u *Upstream) Healthy() bool {
 	healthy := u.healthy()
 	if healthy && u.healthCheckPolicy != nil {
-		healthy = u.Host.Fails() < u.healthCheckPolicy.MaxFails
+		healthy = u.Host().Fails() < u.healthCheckPolicy.MaxFails
 	}
 	if healthy && u.cb != nil {
 		healthy = u.cb.OK()
@@ -94,7 +92,7 @@ func (u *Upstream) Healthy() bool {
 // Full returns true if the remote host
 // cannot receive more requests at this time.
 func (u *Upstream) Full() bool {
-	return u.MaxRequests > 0 && u.Host.NumRequests() >= u.MaxRequests
+	return u.MaxRequests > 0 && u.Host().NumRequests() >= u.MaxRequests
 }
 
 // fillDialInfo returns a filled DialInfo for upstream u, using the request
@@ -124,13 +122,9 @@ func (u *Upstream) fillDialInfo(r *http.Request) (DialInfo, error) {
 	}, nil
 }
 
-func (u *Upstream) fillHost() {
-	host := new(Host)
-	existingHost, loaded := hosts.LoadOrStore(u.String(), host)
-	if loaded {
-		host = existingHost
-	}
-	u.Host = host
+func (u *Upstream) Host() *Host {
+	host, _ := hosts.LoadOrStore(u.String(), new(Host))
+	return host
 }
 
 // Host is the basic, in-memory representation of the state of a remote host.
diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go
index b65dce72..03274009 100644
--- a/modules/caddyhttp/reverseproxy/reverseproxy.go
+++ b/modules/caddyhttp/reverseproxy/reverseproxy.go
@@ -455,14 +455,11 @@ func (h *Handler) proxyLoopIteration(r *http.Request, origReq *http.Request, w h
 	// get the updated list of upstreams
 	upstreams := h.Upstreams
 	if h.DynamicUpstreams != nil {
-		dUpstreams, err := h.DynamicUpstreams.GetUpstreams(r)
+		dUpstreams, err := h.DynamicUpstreams.GetUpstreams(r, h.provisionUpstream)
 		if err != nil {
 			h.logger.Error("failed getting dynamic upstreams; falling back to static upstreams", zap.Error(err))
 		} else {
 			upstreams = dUpstreams
-			for _, dUp := range dUpstreams {
-				h.provisionUpstream(dUp)
-			}
 			h.logger.Debug("provisioned dynamic upstreams", zap.Int("count", len(dUpstreams)))
 			defer func() {
 				// these upstreams are dynamic, so they are only used for this iteration
@@ -503,14 +500,17 @@ func (h *Handler) proxyLoopIteration(r *http.Request, origReq *http.Request, w h
 	// or satisfactorily represented in a URL
 	caddyhttp.SetVar(r.Context(), dialInfoVarKey, dialInfo)
 
+	// host info
+	host := upstream.Host()
+
 	// set placeholders with information about this upstream
 	repl.Set("http.reverse_proxy.upstream.address", dialInfo.String())
 	repl.Set("http.reverse_proxy.upstream.hostport", dialInfo.Address)
 	repl.Set("http.reverse_proxy.upstream.host", dialInfo.Host)
 	repl.Set("http.reverse_proxy.upstream.port", dialInfo.Port)
-	repl.Set("http.reverse_proxy.upstream.requests", upstream.Host.NumRequests())
+	repl.Set("http.reverse_proxy.upstream.requests", host.NumRequests())
 	repl.Set("http.reverse_proxy.upstream.max_requests", upstream.MaxRequests)
-	repl.Set("http.reverse_proxy.upstream.fails", upstream.Host.Fails())
+	repl.Set("http.reverse_proxy.upstream.fails", host.Fails())
 
 	// mutate request headers according to this upstream;
 	// because we're in a retry loop, we have to copy
@@ -736,9 +736,10 @@ func (h Handler) addForwardedHeaders(req *http.Request) error {
 // (This method is mostly the beginning of what was borrowed from the net/http/httputil package in the
 // Go standard library which was used as the foundation.)
 func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origReq *http.Request, repl *caddy.Replacer, di DialInfo, next caddyhttp.Handler) error {
-	_ = di.Upstream.Host.countRequest(1)
+	host := di.Upstream.Host()
+	_ = host.countRequest(1)
 	//nolint:errcheck
-	defer di.Upstream.Host.countRequest(-1)
+	defer host.countRequest(-1)
 
 	// point the request to this upstream
 	h.directRequest(req, di)
@@ -1064,9 +1065,6 @@ func (Handler) directRequest(req *http.Request, di DialInfo) {
 }
 
 func (h Handler) provisionUpstream(upstream *Upstream) {
-	// create or get the host representation for this upstream
-	upstream.fillHost()
-
 	// give it the circuit breaker, if any
 	upstream.cb = h.CB
 
@@ -1300,10 +1298,11 @@ type Selector interface {
 // across successive calls. If the list of upstreams changes or the
 // ordering is not stable, load balancing will suffer. This function
 // may be called during each retry, multiple times per request, and as
-// such, needs to be instantaneous. The returned slice will not be
+// such, needs to be instantaneous. The second argument is a provisioning
+// function to be called with new instances. The returned slice will not be
 // modified.
 type UpstreamSource interface {
-	GetUpstreams(*http.Request) ([]*Upstream, error)
+	GetUpstreams(*http.Request, func(*Upstream)) ([]*Upstream, error)
 }
 
 // Hop-by-hop headers. These are removed when sent to the backend.
diff --git a/modules/caddyhttp/reverseproxy/selectionpolicies.go b/modules/caddyhttp/reverseproxy/selectionpolicies.go
index bc6de351..5f492e85 100644
--- a/modules/caddyhttp/reverseproxy/selectionpolicies.go
+++ b/modules/caddyhttp/reverseproxy/selectionpolicies.go
@@ -255,10 +255,11 @@ func (LeastConnSelection) Select(pool UpstreamPool, _ *http.Request, _ http.Resp
 	var count int
 	leastReqs := -1
 
-	for _, host := range pool {
-		if !host.Available() {
+	for _, upstream := range pool {
+		if !upstream.Available() {
 			continue
 		}
+		host := upstream.Host()
 		numReqs := host.NumRequests()
 		if leastReqs == -1 || numReqs < leastReqs {
 			leastReqs = numReqs
@@ -270,7 +271,7 @@ func (LeastConnSelection) Select(pool UpstreamPool, _ *http.Request, _ http.Resp
 		if numReqs == leastReqs {
 			count++
 			if count > 1 || (weakrand.Int()%count) == 0 { //nolint:gosec
-				bestHost = host
+				bestHost = upstream
 			}
 		}
 	}
@@ -775,7 +776,8 @@ func leastRequests(upstreams []*Upstream) *Upstream {
 		if upstream == nil {
 			continue
 		}
-		reqs := upstream.NumRequests()
+		host := upstream.Host()
+		reqs := host.NumRequests()
 		if reqs == 0 {
 			return upstream
 		}
diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go
index 19e4f3cb..442484fc 100644
--- a/modules/caddyhttp/reverseproxy/upstreams.go
+++ b/modules/caddyhttp/reverseproxy/upstreams.go
@@ -105,7 +105,7 @@ func (su *SRVUpstreams) Provision(ctx caddy.Context) error {
 	return nil
 }
 
-func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
+func (su SRVUpstreams) GetUpstreams(r *http.Request, provision func(*Upstream)) ([]*Upstream, error) {
 	suAddr, service, proto, name := su.expandedAddr(r)
 
 	// first, use a cheap read-lock to return a cached result quickly
@@ -153,6 +153,7 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 			zap.Uint16("weight", rec.Weight))
 		addr := net.JoinHostPort(rec.Target, strconv.Itoa(int(rec.Port)))
 		upstreams[i] = &Upstream{Dial: addr}
+		provision(upstreams[i])
 	}
 
 	// before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
@@ -293,7 +294,7 @@ func (au *AUpstreams) Provision(_ caddy.Context) error {
 	return nil
 }
 
-func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
+func (au AUpstreams) GetUpstreams(r *http.Request, provision func(*Upstream)) ([]*Upstream, error) {
 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
 
 	resolveIpv4 := au.Versions.IPv4 == nil || *au.Versions.IPv4
@@ -352,6 +353,7 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 		upstreams[i] = &Upstream{
 			Dial: net.JoinHostPort(ip.String(), port),
 		}
+		provision(upstreams[i])
 	}
 
 	// before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
@@ -429,7 +431,7 @@ func (mu *MultiUpstreams) Provision(ctx caddy.Context) error {
 	return nil
 }
 
-func (mu MultiUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
+func (mu MultiUpstreams) GetUpstreams(r *http.Request, provision func(*Upstream)) ([]*Upstream, error) {
 	var upstreams []*Upstream
 
 	for i, src := range mu.sources {
@@ -439,7 +441,7 @@ func (mu MultiUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 		default:
 		}
 
-		up, err := src.GetUpstreams(r)
+		up, err := src.GetUpstreams(r, provision)
 		if err != nil {
 			mu.logger.Error("upstream source returned error",
 				zap.Int("source_idx", i),

```
@kkroo Can you provide a stack trace using the original code? So we can be sure we get a fix properly.
stack traces with the original code:

```
  | ==================
  | WARNING: DATA RACE
  | Read at 0x00c000c64ff0 by goroutine 1022:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Healthy()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:86 +0x9d
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Available()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:77 +0x44
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.selectRandomHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:749 +0xe4
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.RandomSelection.Select()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:66 +0x66
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*RandomSelection).Select()
  |       <autogenerated>:1 +0x106
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:478 +0x5c1
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
  |
  | Previous write at 0x00c000c64ff0 by goroutine 1019:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e


  | ==================
  | WARNING: DATA RACE
  | Write at 0x00c000c64ff0 by goroutine 486:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
  |
  |
  | Previous read at 0x00c000c64ff0 by goroutine 483:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).reverseProxy()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:739 +0xa5
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:527 +0x1746
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e


  | ==================
  | WARNING: DATA RACE
  | Write at 0x00c000c64ff0 by goroutine 482:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
  |
  |
  | Previous read at 0x00c000c64ff0 by goroutine 483:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:511 +0x113b
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e


  | ==================
  | WARNING: DATA RACE
  | Write at 0x00c000c65018 by goroutine 467:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1091 +0x326
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
  |
  |
  | Previous read at 0x00c000c65018 by goroutine 463:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Healthy()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:85 +0x6e
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Available()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:77 +0x44
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.selectRandomHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:749 +0xe4
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.RandomSelection.Select()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:66 +0x66
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*RandomSelection).Select()
  |       <autogenerated>:1 +0x106
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:478 +0x5c1
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e



  | ==================
  | WARNING: DATA RACE
  | Write at 0x00c000c65020 by goroutine 467:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1071 +0x8b
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
  |
  |
  | Previous read at 0x00c000c65020 by goroutine 463:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Healthy()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:88 +0x12b
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Available()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:77 +0x44
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.selectRandomHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:749 +0xe4
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.RandomSelection.Select()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:66 +0x66
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*RandomSelection).Select()
  |       <autogenerated>:1 +0x106
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:478 +0x5c1
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e


  | ==================
  | WARNING: DATA RACE
  | Write at 0x00c0000670f8 by goroutine 425:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1080 +0x19b
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
  |
  |
  | Previous write at 0x00c0000670f8 by goroutine 422:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1080 +0x19b
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e



  | ==================
  | WARNING: DATA RACE
  | Write at 0x00c000c64ff0 by goroutine 425:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
  |
  |
  | Previous read at 0x00c000c64ff0 by goroutine 422:
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).reverseProxy()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:741 +0xf5
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:527 +0x1746
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
  |   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
  |       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
```
@kkroo Thanks, that makes more sense. What is your config?

The stack trace (and the issue title, d'oh) indicates that dynamic upstreams are being used. Since that is the case I need to know your config so I can hunt down the solution to the race.

Please post the exact config without changes/redactions so I can accurately create a fix. Thank you!
Thank you for looking into this @mholt 
The config is really large since i am generating them using a custom config provider. here is the reverse proxy handler snippet unchanged:

```
                {
                  "dynamic_upstreams": {
                    "name": "service.upstream.dev",
                    "port": "80",
                    "source": "a",
                    "versions": {
                      "ipv4": true,
                      "ipv6": true
                    }
                  },
                  "handler": "reverse_proxy",
                  "headers": {
                    "request": {
                      "set": {
                        "Host": [
                          "service.upstream.dev:80"
                        ],
                        "X-Forwarded-For": [
                          "{http.request.remote}"
                        ],
                        "X-Forwarded-Host": [
                          "{http.request.hostport}"
                        ],
                        "X-Forwarded-Proto": [
                          "http"
                        ],
                        "X-Real-Ip": [
                          "{http.request.remote}"
                        ]
                      }
                    }
                  },
                  "health_checks": {
                    "passive": {
                      "fail_duration": 30000000000,
                      "max_fails": 5,
                      "unhealthy_status": [
                        501,
                        503
                      ]
                    }
                  },
                  "load_balancing": {
                    "retries": 5,
                    "try_duration": 30000000000
                  },
                  "transport": {
                    "keep_alive": {
                      "idle_timeout": 15000000000,
                      "max_idle_conns_per_host": 1000,
                      "probe_interval": 1000000000
                    },
                    "protocol": "http",
                    "read_timeout": 30000000000
                  },
                  "trusted_proxies": [
                    "::1/128",
                    "127.0.0.1/32",
                    "10.0.0.0/8",
                    "172.16.0.0/12",
                    "192.168.0.0/16"
                  ]
                }
```
Thanks. That helps too.

We're actually creating new `*Upstream` values for every call to `GetUpstreams`, which is correct. It can't be a race then because different requests see different `*Upstream` values.

Except if they're cached.

Do you have a moment to test if the race still occurs if you comment out these lines:

https://github.com/caddyserver/caddy/blob/d6f86cccf5fa5b4eb30141da390cf2439746c5da/modules/caddyhttp/reverseproxy/upstreams.go#L323-L329

i.e. if we don't use a cached result from another goroutine, maybe the race goes away.

And if so, then I think a better solution is to use `Upstream` instead of `*Upstream`, if possible.
I applied the following diff:
```
diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go
index 19e4f3cb..4c9f5c1b 100644
--- a/modules/caddyhttp/reverseproxy/upstreams.go
+++ b/modules/caddyhttp/reverseproxy/upstreams.go
@@ -320,12 +320,12 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 	auStr := repl.ReplaceAll(au.String()+ipVersion, "")
 
 	// first, use a cheap read-lock to return a cached result quickly
-	aAaaaMu.RLock()
-	cached := aAaaa[auStr]
-	aAaaaMu.RUnlock()
-	if cached.isFresh() {
-		return cached.upstreams, nil
-	}
+	//aAaaaMu.RLock()
+	//cached := aAaaa[auStr]
+	//aAaaaMu.RUnlock()
+	//if cached.isFresh() {
+	//	return cached.upstreams, nil
+	//}
 
 	// otherwise, obtain a write-lock to update the cached value
 	aAaaaMu.Lock()
@@ -334,10 +334,10 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 	// check to see if it's still stale, since we're now in a different
 	// lock from when we first checked freshness; another goroutine might
 	// have refreshed it in the meantime before we re-obtained our lock
-	cached = aAaaa[auStr]
-	if cached.isFresh() {
-		return cached.upstreams, nil
-	}
+	//cached = aAaaa[auStr]
+	//if cached.isFresh() {
+	//	return cached.upstreams, nil
+	//}
 
 	name := repl.ReplaceAll(au.Name, "")
 	port := repl.ReplaceAll(au.Port, "")
@@ -355,12 +355,12 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 	}
 
 	// before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
-	if cached.freshness.IsZero() && len(srvs) >= 100 {
-		for randomKey := range aAaaa {
-			delete(aAaaa, randomKey)
-			break
-		}
-	}
+	//if cached.freshness.IsZero() && len(srvs) >= 100 {
+	//	for randomKey := range aAaaa {
+	//		delete(aAaaa, randomKey)
+	//		break
+	//	}
+	//}
 
 	aAaaa[auStr] = aLookup{
 		aUpstreams: au,
```


These are the races I am still getting:

```
| ==================
| WARNING: DATA RACE
| Write at 0x00c0009f8690 by goroutine 436:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
|
| Previous read at 0x00c0009f8690 by goroutine 427:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).reverseProxy()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:741 +0xf5
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:527 +0x1746
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
| ==================
| WARNING: DATA RACE
| Write at 0x00c0009f86c0 by goroutine 436:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1071 +0x8b
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
|
| Previous write at 0x00c0009f86c0 by goroutine 427:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1071 +0x8b
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

| ==================
| ==================
| WARNING: DATA RACE
| Write at 0x00c000067db8 by goroutine 436:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1080 +0x19b
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
|
| Previous write at 0x00c000067db8 by goroutine 427:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1080 +0x19b
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
| ==================
| ==================
| WARNING: DATA RACE
| Write at 0x00c0009f86b8 by goroutine 436:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1091 +0x326
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
|
| Previous write at 0x00c0009f86b8 by goroutine 427:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1091 +0x326
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

| ==================
| WARNING: DATA RACE
| Write at 0x00c0009f86c0 by goroutine 438:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1071 +0x8b
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
|
| Previous read at 0x00c0009f86c0 by goroutine 436:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Healthy()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:88 +0x12b
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Available()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:77 +0x44
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.selectRandomHost()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:749 +0xe4
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.RandomSelection.Select()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:66 +0x66
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*RandomSelection).Select()
|       <autogenerated>:1 +0x106
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:478 +0x5c1
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

| ==================
| ==================
| WARNING: DATA RACE
| Write at 0x00c0009f86b8 by goroutine 438:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1091 +0x326
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
|
| Previous read at 0x00c0009f86b8 by goroutine 436:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Healthy()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:85 +0x6e
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).Available()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:77 +0x44
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.selectRandomHost()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:749 +0xe4
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.RandomSelection.Select()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/selectionpolicies.go:66 +0x66
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*RandomSelection).Select()
|       <autogenerated>:1 +0x106
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:478 +0x5c1
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

| ==================
| ==================
| WARNING: DATA RACE
| Write at 0x00c0009f8690 by goroutine 456:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
|
| Previous read at 0x00c0009f8690 by goroutine 443:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:511 +0x113b
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

| ==================
| ==================
| WARNING: DATA RACE
| Write at 0x00c0009f8690 by goroutine 668:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e

|
|
| Previous write at 0x00c0009f8690 by goroutine 665:
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Upstream).fillHost()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/hosts.go:133 +0x15c
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.Handler.provisionUpstream()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:1068 +0x56
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).proxyLoopIteration()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:464 +0x21d7
|   github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy.(*Handler).ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/reverseproxy.go:434 +0x9ac
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.wrapMiddleware.func1.1()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/routes.go:331 +0xe8
|   github.com/caddyserver/caddy/v2/modules/caddyhttp.HandlerFunc.ServeHTTP()
|       /root/go/src/github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyhttp.go:58 +0x7e
```
i take that back. was building from a vendored path so these changes didn't get picked up. i updated sources with the patch and that does seem to fix the race
Phew :pray: 

After I saw your update last night, I was struggling to figure out how it could possibly still be racey. I'm really glad it actually does fix it :sweat_smile: 

Ok, so we know a solution; I will try to get it fixed with minimal intrusion ASAP. Thank you!

Fix patch:
diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go
index 249326dd932..f11c8e33a6b 100644
--- a/modules/caddyhttp/reverseproxy/reverseproxy.go
+++ b/modules/caddyhttp/reverseproxy/reverseproxy.go
@@ -356,6 +356,7 @@ func (h *Handler) Provision(ctx caddy.Context) error {
 	if h.HealthChecks != nil {
 		// set defaults on passive health checks, if necessary
 		if h.HealthChecks.Passive != nil {
+			h.HealthChecks.Passive.logger = h.logger.Named("health_checker.passive")
 			if h.HealthChecks.Passive.FailDuration > 0 && h.HealthChecks.Passive.MaxFails == 0 {
 				h.HealthChecks.Passive.MaxFails = 1
 			}
@@ -1077,12 +1078,11 @@ func (h Handler) provisionUpstream(upstream *Upstream) {
 	// without MaxRequests), copy the value into this upstream, since the
 	// value in the upstream (MaxRequests) is what is used during
 	// availability checks
-	if h.HealthChecks != nil && h.HealthChecks.Passive != nil {
-		h.HealthChecks.Passive.logger = h.logger.Named("health_checker.passive")
-		if h.HealthChecks.Passive.UnhealthyRequestCount > 0 &&
-			upstream.MaxRequests == 0 {
-			upstream.MaxRequests = h.HealthChecks.Passive.UnhealthyRequestCount
-		}
+	if h.HealthChecks != nil &&
+		h.HealthChecks.Passive != nil &&
+		h.HealthChecks.Passive.UnhealthyRequestCount > 0 &&
+		upstream.MaxRequests == 0 {
+		upstream.MaxRequests = h.HealthChecks.Passive.UnhealthyRequestCount
 	}
 
 	// upstreams need independent access to the passive
diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go
index 8bdc60dd5ff..09efdd5186a 100644
--- a/modules/caddyhttp/reverseproxy/upstreams.go
+++ b/modules/caddyhttp/reverseproxy/upstreams.go
@@ -114,7 +114,7 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 	cached := srvs[suAddr]
 	srvsMu.RUnlock()
 	if cached.isFresh() {
-		return cached.upstreams, nil
+		return allNew(cached.upstreams), nil
 	}
 
 	// otherwise, obtain a write-lock to update the cached value
@@ -126,7 +126,7 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 	// have refreshed it in the meantime before we re-obtained our lock
 	cached = srvs[suAddr]
 	if cached.isFresh() {
-		return cached.upstreams, nil
+		return allNew(cached.upstreams), nil
 	}
 
 	su.logger.Debug("refreshing SRV upstreams",
@@ -145,7 +145,7 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 		su.logger.Warn("SRV records filtered", zap.Error(err))
 	}
 
-	upstreams := make([]*Upstream, len(records))
+	upstreams := make([]Upstream, len(records))
 	for i, rec := range records {
 		su.logger.Debug("discovered SRV record",
 			zap.String("target", rec.Target),
@@ -153,7 +153,7 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 			zap.Uint16("priority", rec.Priority),
 			zap.Uint16("weight", rec.Weight))
 		addr := net.JoinHostPort(rec.Target, strconv.Itoa(int(rec.Port)))
-		upstreams[i] = &Upstream{Dial: addr}
+		upstreams[i] = Upstream{Dial: addr}
 	}
 
 	// before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
@@ -170,7 +170,7 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 		upstreams:    upstreams,
 	}
 
-	return upstreams, nil
+	return allNew(upstreams), nil
 }
 
 func (su SRVUpstreams) String() string {
@@ -206,7 +206,7 @@ func (SRVUpstreams) formattedAddr(service, proto, name string) string {
 type srvLookup struct {
 	srvUpstreams SRVUpstreams
 	freshness    time.Time
-	upstreams    []*Upstream
+	upstreams    []Upstream
 }
 
 func (sl srvLookup) isFresh() bool {
@@ -325,7 +325,7 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 	cached := aAaaa[auStr]
 	aAaaaMu.RUnlock()
 	if cached.isFresh() {
-		return cached.upstreams, nil
+		return allNew(cached.upstreams), nil
 	}
 
 	// otherwise, obtain a write-lock to update the cached value
@@ -337,7 +337,7 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 	// have refreshed it in the meantime before we re-obtained our lock
 	cached = aAaaa[auStr]
 	if cached.isFresh() {
-		return cached.upstreams, nil
+		return allNew(cached.upstreams), nil
 	}
 
 	name := repl.ReplaceAll(au.Name, "")
@@ -348,15 +348,15 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 		return nil, err
 	}
 
-	upstreams := make([]*Upstream, len(ips))
+	upstreams := make([]Upstream, len(ips))
 	for i, ip := range ips {
-		upstreams[i] = &Upstream{
+		upstreams[i] = Upstream{
 			Dial: net.JoinHostPort(ip.String(), port),
 		}
 	}
 
 	// before adding a new one to the cache (as opposed to replacing stale one), make room if cache is full
-	if cached.freshness.IsZero() && len(srvs) >= 100 {
+	if cached.freshness.IsZero() && len(aAaaa) >= 100 {
 		for randomKey := range aAaaa {
 			delete(aAaaa, randomKey)
 			break
@@ -369,7 +369,7 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) {
 		upstreams:  upstreams,
 	}
 
-	return upstreams, nil
+	return allNew(upstreams), nil
 }
 
 func (au AUpstreams) String() string { return net.JoinHostPort(au.Name, au.Port) }
@@ -377,7 +377,7 @@ func (au AUpstreams) String() string { return net.JoinHostPort(au.Name, au.Port)
 type aLookup struct {
 	aUpstreams AUpstreams
 	freshness  time.Time
-	upstreams  []*Upstream
+	upstreams  []Upstream
 }
 
 func (al aLookup) isFresh() bool {
@@ -483,6 +483,14 @@ func (u *UpstreamResolver) ParseAddresses() error {
 	return nil
 }
 
+func allNew(upstreams []Upstream) []*Upstream {
+	results := make([]*Upstream, len(upstreams))
+	for i := range upstreams {
+		results[i] = &Upstream{Dial: upstreams[i].Dial}
+	}
+	return results
+}
+
 var (
 	srvs   = make(map[string]srvLookup)
 	srvsMu sync.RWMutex


Test patch: