Repository: caddyserver/caddy (language: go)

Issue:
Enabling get_certificate should disable cert automation
 I set the get_certificate section in caddyfile, caddy verified all parameters and directly goes to HTTP-01 challenge. 

here's my procedure to reproduce the issue:

1. pull the latest version of caddy image and create a fileserver, and serve /etc/caddy as root 
```
docker pull caddy
docker run --rm --hostname cert.mysite -p 80:80  -p 443:443 -it --name certmgr caddy caddy file-server --root /etc/caddy --debug
```
```
root@demo:/test# docker run --rm --hostname cert.mysite -p 80:80  -p 443:443 -it --name certmgr caddy caddy file-server --root /etc/caddy --debug
2023/03/03 09:50:13.498	WARN	admin	admin endpoint disabled
2023/03/03 09:50:13.499	INFO	tls.cache.maintenance	started background certificate maintenance	{"cache": "0xc0003fae00"}
2023/03/03 09:50:13.499	WARN	http	server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server	{"server_name": "static", "http_port": 80}
2023/03/03 09:50:13.499	INFO	tls	cleaning storage unit	{"description": "FileStorage:/data/caddy"}
2023/03/03 09:50:13.499	INFO	tls	finished cleaning storage units
2023/03/03 09:50:13.499	DEBUG	http	starting server loop	{"address": "[::]:80", "tls": false, "http3": false}
2023/03/03 09:50:13.499	INFO	http.log	server running	{"name": "static", "protocols": ["h1", "h2", "h3"]}
2023/03/03 09:50:13.499	INFO	Caddy serving static files on :80
```
2.  open another terminal and copy the hosts file from cert.mysite
```
mkdir -p /test && cd /test
docker cp certmgr:/etc/hosts /test/hosts
```
3. try to validate if http://cert.mysite/Caddyfile is avaliable in another container:
```
root@demo:/test# docker run --rm -it -v /test/hosts:/etc/hosts -v /test/Caddyfile:/etc/caddy/Caddyfile caddy wget -o /dev/null  -O - cert.mysite/Caddyfile
# The Caddyfile is an easy way to configure your Caddy web server.
#
# Unless the file starts with a global options block, the first
# uncommented line is always the address of your site.
#
# To use your own domain name (with automatic HTTPS), first make
# sure your domain's A/AAAA DNS records are properly pointed to
# this machine's public IP, then replace ":80" below with your
# domain name.

:80 {
	# Set this path to your site's directory.
	root * /usr/share/caddy

	# Enable the static file server.
	file_server

	# Another common task is to set up a reverse proxy:
	# reverse_proxy localhost:8080

	# Or serve a PHP site through php-fpm:
	# php_fastcgi localhost:9000
}

# Refer to the Caddy docs for more information:
# https://caddyserver.com/docs/caddyfile
```
one record of log is printed by cert.mysite
```
root@demo:/test# docker run --rm --hostname cert.mysite -p 80:80  -p 443:443 -it --name certmgr caddy caddy file-server --root /etc/caddy --debug
2023/03/03 09:50:13.498	WARN	admin	admin endpoint disabled
2023/03/03 09:50:13.499	INFO	tls.cache.maintenance	started background certificate maintenance	{"cache": "0xc0003fae00"}
2023/03/03 09:50:13.499	WARN	http	server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server	{"server_name": "static", "http_port": 80}
2023/03/03 09:50:13.499	INFO	tls	cleaning storage unit	{"description": "FileStorage:/data/caddy"}
2023/03/03 09:50:13.499	INFO	tls	finished cleaning storage units
2023/03/03 09:50:13.499	DEBUG	http	starting server loop	{"address": "[::]:80", "tls": false, "http3": false}
2023/03/03 09:50:13.499	INFO	http.log	server running	{"name": "static", "protocols": ["h1", "h2", "h3"]}
2023/03/03 09:50:13.499	INFO	Caddy serving static files on :80
2023/03/03 09:50:19.583	DEBUG	http.handlers.file_server	sanitized path join	{"site_root": "/etc/caddy", "request_path": "/Caddyfile", "result": "/etc/caddy/Caddyfile"}
2023/03/03 09:50:19.583	DEBUG	http.handlers.file_server	opening file	{"filename": "/etc/caddy/Caddyfile"}
```

4. create /test/Caddyfile with:
```
{
	debug
}
https://web.mysite {
	tls {
		get_certificate http http://cert.mysite/cert
	}
	file_server
	root * /usr/share/caddy
}
```
5. star anoter container for HTTP cert gatter:
```
root@demo:/test# docker run --rm -it -v /test/hosts:/etc/hosts -v /test/Caddyfile:/etc/caddy/Caddyfile caddy
2023/03/03 09:54:56.848	INFO	using provided configuration	{"config_file": "/etc/caddy/Caddyfile", "config_adapter": "caddyfile"}
2023/03/03 09:54:56.849	INFO	admin	admin endpoint started	{"address": "localhost:2019", "enforce_origin": false, "origins": ["//[::1]:2019", "//127.0.0.1:2019", "//localhost:2019"]}
2023/03/03 09:54:56.850	INFO	http	server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS	{"server_name": "srv0", "https_port": 443}
2023/03/03 09:54:56.850	INFO	http	enabling automatic HTTP->HTTPS redirects	{"server_name": "srv0"}
2023/03/03 09:54:56.850	INFO	tls.cache.maintenance	started background certificate maintenance	{"cache": "0xc000119260"}
2023/03/03 09:54:56.850	INFO	tls	cleaning storage unit	{"description": "FileStorage:/data/caddy"}
2023/03/03 09:54:56.851	INFO	tls	finished cleaning storage units
2023/03/03 09:54:56.851	INFO	http	enabling HTTP/3 listener	{"addr": ":443"}
2023/03/03 09:54:56.851	INFO	failed to sufficiently increase receive buffer size (was: 208 kiB, wanted: 2048 kiB, got: 416 kiB). See https://github.com/quic-go/quic-go/wiki/UDP-Receive-Buffer-Size for details.
2023/03/03 09:54:56.851	DEBUG	http	starting server loop	{"address": "[::]:443", "tls": true, "http3": true}
2023/03/03 09:54:56.851	INFO	http.log	server running	{"name": "srv0", "protocols": ["h1", "h2", "h3"]}
2023/03/03 09:54:56.851	DEBUG	http	starting server loop	{"address": "[::]:80", "tls": false, "http3": false}
2023/03/03 09:54:56.851	INFO	http.log	server running	{"name": "remaining_auto_https_redirects", "protocols": ["h1", "h2", "h3"]}
2023/03/03 09:54:56.851	INFO	http	enabling automatic TLS certificate management	{"domains": ["web.mysite"]}
2023/03/03 09:54:56.851	INFO	autosaved config (load with --resume flag)	{"file": "/config/caddy/autosave.json"}
2023/03/03 09:54:56.851	INFO	serving initial configuration
2023/03/03 09:54:56.852	INFO	tls.obtain	acquiring lock	{"identifier": "web.mysite"}
2023/03/03 09:54:56.852	INFO	tls.obtain	lock acquired	{"identifier": "web.mysite"}
2023/03/03 09:54:56.852	INFO	tls.obtain	obtaining certificate	{"identifier": "web.mysite"}
2023/03/03 09:54:56.853	DEBUG	events	event	{"name": "cert_obtaining", "id": "5ee698ea-a7b0-42d4-8fd2-2d148a1bcf91", "origin": "tls", "data": {"identifier":"web.mysite"}}
2023/03/03 09:54:56.853	DEBUG	tls.obtain	trying issuer 1/2	{"issuer": "acme-v02.api.letsencrypt.org-directory"}
2023/03/03 09:54:57.428	DEBUG	tls.acme_client	http request	{"method": "GET", "url": "https://acme-v02.api.letsencrypt.org/directory", "headers": {"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]}, "response_headers": {"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["756"],"Content-Type":["application/json"],"Date":["Fri, 03 Mar 2023 09:54:57 GMT"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]}, "status_code": 200}
2023/03/03 09:54:57.607	DEBUG	tls.acme_client	http request	{"method": "HEAD", "url": "https://acme-v02.api.letsencrypt.org/acme/new-nonce", "headers": {"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]}, "response_headers": {"Cache-Control":["public, max-age=0, no-cache"],"Date":["Fri, 03 Mar 2023 09:54:57 GMT"],"Link":["<https://acme-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Replay-Nonce":["F977mkR7mwCLcOasyyIoTtnnqzTE5wOhhDe4xtpAylAi_Pc"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]}, "status_code": 200}
2023/03/03 09:54:57.797	DEBUG	tls.acme_client	http request	{"method": "POST", "url": "https://acme-v02.api.letsencrypt.org/acme/new-acct", "headers": {"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]}, "response_headers": {"Boulder-Requester":["991443716"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["267"],"Content-Type":["application/json"],"Date":["Fri, 03 Mar 2023 09:54:57 GMT"],"Link":["<https://acme-v02.api.letsencrypt.org/directory>;rel=\"index\"","<https://letsencrypt.org/documents/LE-SA-v1.3-September-21-2022.pdf>;rel=\"terms-of-service\""],"Location":["https://acme-v02.api.letsencrypt.org/acme/acct/991443716"],"Replay-Nonce":["F70EyZ_pC24oN4PtiMWxr1NXVxWGzy0lP6GgbhBSUStj1tw"],"Server":["nginx"],"Strict-Transport-Security":["max-age=604800"],"X-Frame-Options":["DENY"]}, "status_code": 201}
2023/03/03 09:54:57.798	INFO	tls	waiting on internal rate limiter	{"identifiers": ["web.mysite"], "ca": "https://acme-v02.api.letsencrypt.org/directory", "account": ""}
2023/03/03 09:54:57.798	INFO	tls	done waiting on internal rate limiter	{"identifiers": ["web.mysite"], "ca": "https://acme-v02.api.letsencrypt.org/directory", "account": ""}
2023/03/03 09:54:57.978	DEBUG	tls.acme_client	http request	{"method": "POST", "url": "https://acme-v02.api.letsencrypt.org/acme/new-order", "headers": {"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]}, "response_headers": {"Boulder-Requester":["991443716"],"Cache-Control":["public, max-age=0, no-cache"],"Content-Length":["212"],"Content-Type":["application/problem+json"],"Date":["Fri, 03 Mar 2023 09:54:57 GMT"],"Link":["<https://acme-v02.api.letsencrypt.org/directory>;rel=\"index\""],"Replay-Nonce":["F70EZ0AyzmBi1gJHVQOnVwxHCzrDIRBTmLFAHCqPZh1mMWo"],"Server":["nginx"]}, "status_code": 400}
2023/03/03 09:54:57.978	ERROR	tls.obtain	could not get certificate from issuer	{"identifier": "web.mysite", "issuer": "acme-v02.api.letsencrypt.org-directory", "error": "HTTP 400 urn:ietf:params:acme:error:rejectedIdentifier - Error creating new order :: Cannot issue for \"web.mysite\": Domain name does not end with a valid public suffix (TLD)"}
2023/03/03 09:54:57.978	DEBUG	tls.obtain	trying issuer 2/2	{"issuer": "acme.zerossl.com-v2-DV90"}
2023/03/03 09:54:57.978	WARN	tls	missing email address for ZeroSSL; it is strongly recommended to set one for next time
2023/03/03 09:54:59.349	INFO	tls	generated EAB credentials	{"key_id": "LLRR_JiXvnP1Gyxq8z_DJg"}
2023/03/03 09:55:00.338	DEBUG	tls.acme_client	http request	{"method": "GET", "url": "https://acme.zerossl.com/v2/DV90", "headers": {"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]}, "response_headers": {"Access-Control-Allow-Origin":["*"],"Content-Length":["645"],"Content-Type":["application/json"],"Date":["Fri, 03 Mar 2023 09:55:00 GMT"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]}, "status_code": 200}
2023/03/03 09:55:01.042	DEBUG	tls.acme_client	http request	{"method": "HEAD", "url": "https://acme.zerossl.com/v2/DV90/newNonce", "headers": {"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]}, "response_headers": {"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Type":["application/octet-stream"],"Date":["Fri, 03 Mar 2023 09:55:00 GMT"],"Link":["<https://acme.zerossl.com/v2/DV90>;rel=\"index\""],"Replay-Nonce":["choyu-v75RZ-bY957c7mtuaNcSeoRrEvG7HmRHJ7_Us"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]}, "status_code": 200}
2023/03/03 09:55:01.864	DEBUG	tls.acme_client	http request	{"method": "POST", "url": "https://acme.zerossl.com/v2/DV90/newAccount", "headers": {"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]}, "response_headers": {"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Length":["579"],"Content-Type":["application/json"],"Date":["Fri, 03 Mar 2023 09:55:01 GMT"],"Link":["<https://acme.zerossl.com/v2/DV90>;rel=\"index\""],"Location":["https://acme.zerossl.com/v2/DV90/account/LLRR_JiXvnP1Gyxq8z_DJg"],"Replay-Nonce":["V4YzFYtZr4O5Kj-c-VBhxNZo5toNBIsaofRmEMM11AM"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]}, "status_code": 201}
2023/03/03 09:55:01.865	INFO	tls	waiting on internal rate limiter	{"identifiers": ["web.mysite"], "ca": "https://acme.zerossl.com/v2/DV90", "account": ""}
2023/03/03 09:55:01.865	INFO	tls	done waiting on internal rate limiter	{"identifiers": ["web.mysite"], "ca": "https://acme.zerossl.com/v2/DV90", "account": ""}
2023/03/03 09:55:02.512	DEBUG	tls.acme_client	http request	{"method": "POST", "url": "https://acme.zerossl.com/v2/DV90/newOrder", "headers": {"Content-Type":["application/jose+json"],"User-Agent":["Caddy/2.6.4 CertMagic acmez (linux; amd64)"]}, "response_headers": {"Access-Control-Allow-Origin":["*"],"Cache-Control":["max-age=0, no-cache, no-store"],"Content-Length":["116"],"Content-Type":["application/problem+json"],"Date":["Fri, 03 Mar 2023 09:55:02 GMT"],"Link":["<https://acme.zerossl.com/v2/DV90>;rel=\"index\""],"Replay-Nonce":["jQSBi7xGSLNSh3oaSYD3XZqo6XJkKRC4EEuYSOPB2wI"],"Server":["nginx"],"Strict-Transport-Security":["max-age=15724800; includeSubDomains"]}, "status_code": 400}
2023/03/03 09:55:02.512	ERROR	tls.obtain	could not get certificate from issuer	{"identifier": "web.mysite", "issuer": "acme.zerossl.com-v2-DV90", "error": "HTTP 400 urn:ietf:params:acme:error:rejectedIdentifier - Invalid DNS identifier [web.mysite]"}
2023/03/03 09:55:02.512	DEBUG	events	event	{"name": "cert_failed", "id": "1298f96f-180f-4a9e-97c7-ccdc8854f8c0", "origin": "tls", "data": {"error":{},"identifier":"web.mysite","issuers":["acme-v02.api.letsencrypt.org-directory","acme.zerossl.com-v2-DV90"],"renewal":false}}
2023/03/03 09:55:02.512	ERROR	tls.obtain	will retry	{"error": "[web.mysite] Obtain: [web.mysite] creating new order: attempt 1: https://acme.zerossl.com/v2/DV90/newOrder: HTTP 400 urn:ietf:params:acme:error:rejectedIdentifier - Invalid DNS identifier [web.mysite] (ca=https://acme.zerossl.com/v2/DV90)", "attempt": 1, "retrying_in": 60, "elapsed": 5.659666017, "max_duration": 2592000}

```
6. no new log from cert.mysite, let's get the version of caddy in container:
```
root@demo:/test# docker run --rm -it -v /test/hosts:/etc/hosts -v /test/Caddyfile:/etc/caddy/Caddyfile caddy caddy version
v2.6.4 h1:2hwYqiRwk1tf3VruhMpLcYTg+11fCdr8S3jhNAdnPy8=
```



Discussion / hints:
> 

If we change the  /test/Caddyfile in setp 4 with some existing file just like:
```
{
	debug
}
https://web.mysite {
	tls {
		get_certificate http http://cert.mysite/Caddyfile
	}
	file_server
	root * /usr/share/caddy
}
```
It makes no difference. I know manual says the response must have a 200 status code and the body must contain a PEM chain including the full certificate (with intermediates) as well as the private key. But there is no access_log in cert.mysite, which means  cert.mysite didn't get the HTTP request from caddy.
Thanks for the detailed issue!

I suppose the expectation here is to not get certificates from the issuers because a certificate manager (get_ceritificate subdirective) is specified.

But just to clarify, you did actually make an HTTPS request to `https://web.mysite` right? I would expect to see something in the logs about that if you did. Regardless of the issuers trying to get a certificate at startup, you still need to make a TLS handshake with `web.mysite` in order for the certificate manager to be invoked.
> 

My fault, I didn't understand the description about get_certificate . I tried to make a request to https://web.mysite from cert.mysite, and I got these logs from cert.mysite:
```
{"level":"debug","ts":1678158186.21117,"logger":"http.handlers.file_server","msg":"sanitized path join","site_root":"/etc/caddy","request_path":"/cert","result":"/etc/caddy/cert"}
{"level":"debug","ts":1678158186.2112262,"logger":"http.handlers.file_server","msg":"opening file","filename":"/etc/caddy/cert"}

```

It works!  Every time I make a request from cert.mysite to https://web.mysite, these logs are printed. The one last problem is: although HTTP is functional on web.mysite, caddy still keeps requesting for letsencrypt and zerossl.

Thanks to your reply, I find the way to build a external cert manager for caddy now.
Glad you were able to make progress.

I'm currently addressing the "still requesting certs from CAs" issue -- I guess most users would expect, in the presence of a `get_certificate`, that CAs are not even attempted.
> Glad you were able to make progress.
> 
> I'm currently addressing the "still requesting certs from CAs" issue -- I guess most users would expect, in the presence of a `get_certificate`, that CAs are not even attempted.

Thank you!

Actually, I was trying to build something like issuer proxy for caddy. It has two benefit: 1. being able to setup SSL for caddy without internet. 2. No more dns token for every single server without public IP, dns token leaking is far more serious than certificate leaking.

I thought I cloud create a certmgr to manage certificate, and let caddy pull  SSL certificate from certmgr by get_certificate without internet access. Now I realized that by using get_certificate, caddy will request certificate for every single session, and caddy will not manage/cache certificate any more, because the build-in renew service is totally disabled. That makes sense because manual says "certificate manager modules implies that an external tool or service is keeping the certificate renewed". The goal is partly achieved, HTTPS server without internet could be setting up, but if certmgr is down, every downstream caddy server would be unable to initialized new sessions.
You could solve that by running a caching proxy on the same machine as Caddy, or using something like like the `cache-handler` plugin and making another server in Caddy which would cache it.

```
https://example.com {
	tls {
		get_certificate http http://localhost:8881
	}
}

:8881 {
	cache
	reverse_proxy your-cert-server:8080
}
```

Or you could just implement your own module instead of `http` which does caching. It's pluggable, see https://caddyserver.com/docs/json/apps/tls/automation/policies/get_certificate/ which are all the existing ones currently.
> You could solve that by running a caching proxy on the same machine as Caddy, or using something like like the `cache-handler` plugin and making another server in Caddy which would cache it.
> 
> ```
> https://example.com {
> 	tls {
> 		get_certificate http http://localhost:8881
> 	}
> }
> 
> :8881 {
> 	cache
> 	reverse_proxy your-cert-server:8080
> }
> ```
> 
> Or you could just implement your own module instead of `http` which does caching. It's pluggable, see https://caddyserver.com/docs/json/apps/tls/automation/policies/get_certificate/ which are all the existing ones currently.

Thank you! I think cache-handler will be good enough for most cases.
I've been working on this the last few days... it's quite tricky / involved; I've started by making certificate manager config part of OnDemand config in CertMagic: https://github.com/caddyserver/certmagic/compare/master...certmanager-refactor

Fix patch:
diff --git a/caddyconfig/httpcaddyfile/tlsapp.go b/caddyconfig/httpcaddyfile/tlsapp.go
index 2021970bec9..c63569e4184 100644
--- a/caddyconfig/httpcaddyfile/tlsapp.go
+++ b/caddyconfig/httpcaddyfile/tlsapp.go
@@ -218,6 +218,10 @@ func (st ServerType) buildTLSApp(
 			if len(ap.Issuers) == 0 {
 				var internal, external []string
 				for _, s := range ap.SubjectsRaw {
+					// do not create Issuers for Tailscale domains; they will be given a Manager instead
+					if strings.HasSuffix(strings.ToLower(s), ".ts.net") {
+						continue
+					}
 					if !certmagic.SubjectQualifiesForCert(s) {
 						return nil, warnings, fmt.Errorf("subject does not qualify for certificate: '%s'", s)
 					}
diff --git a/go.mod b/go.mod
index 003da3f5d9f..0f15b8645e8 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alecthomas/chroma/v2 v2.7.0
 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
-	github.com/caddyserver/certmagic v0.17.3-0.20230510193943-53140d52202c
+	github.com/caddyserver/certmagic v0.17.3-0.20230511183644-8728b186fa68
 	github.com/dustin/go-humanize v1.0.1
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/google/cel-go v0.14.0
diff --git a/go.sum b/go.sum
index afd511bd11b..778721e939e 100644
--- a/go.sum
+++ b/go.sum
@@ -97,8 +97,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/caddyserver/certmagic v0.17.3-0.20230510193943-53140d52202c h1:pEMS0l8kE/5xxrncv+Qq81fzr29R+zk++E7KAYiyBe4=
-github.com/caddyserver/certmagic v0.17.3-0.20230510193943-53140d52202c/go.mod h1:e0YLTnXIopZ05bBWCLzpIf1Yvk27Q90FGUmGowFRDY8=
+github.com/caddyserver/certmagic v0.17.3-0.20230511183644-8728b186fa68 h1:B3ZszrtQ3ksok0Cby1mGT/4T513vxXpOUCf5862c/NQ=
+github.com/caddyserver/certmagic v0.17.3-0.20230511183644-8728b186fa68/go.mod h1:e0YLTnXIopZ05bBWCLzpIf1Yvk27Q90FGUmGowFRDY8=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
diff --git a/modules/caddyhttp/autohttps.go b/modules/caddyhttp/autohttps.go
index 86b34d3ab4d..4ade3c5bf08 100644
--- a/modules/caddyhttp/autohttps.go
+++ b/modules/caddyhttp/autohttps.go
@@ -83,6 +83,8 @@ func (ahc AutoHTTPSConfig) Skipped(name string, skipSlice []string) bool {
 // even servers to the app, which still need to be set up with the
 // rest of them during provisioning.
 func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) error {
+	logger := app.logger.Named("auto_https")
+
 	// this map acts as a set to store the domain names
 	// for which we will manage certificates automatically
 	uniqueDomainsForCerts := make(map[string]struct{})
@@ -114,13 +116,13 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 			srv.AutoHTTPS = new(AutoHTTPSConfig)
 		}
 		if srv.AutoHTTPS.Disabled {
-			app.logger.Warn("automatic HTTPS is completely disabled for server", zap.String("server_name", srvName))
+			logger.Warn("automatic HTTPS is completely disabled for server", zap.String("server_name", srvName))
 			continue
 		}
 
 		// skip if all listeners use the HTTP port
 		if !srv.listenersUseAnyPortOtherThan(app.httpPort()) {
-			app.logger.Warn("server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server",
+			logger.Warn("server is listening only on the HTTP port, so no automatic HTTPS will be applied to this server",
 				zap.String("server_name", srvName),
 				zap.Int("http_port", app.httpPort()),
 			)
@@ -134,7 +136,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 		// needing to specify one empty policy to enable it
 		if srv.TLSConnPolicies == nil &&
 			!srv.listenersUseAnyPortOtherThan(app.httpsPort()) {
-			app.logger.Info("server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS",
+			logger.Info("server is listening only on the HTTPS port but has no TLS connection policies; adding one to enable TLS",
 				zap.String("server_name", srvName),
 				zap.Int("https_port", app.httpsPort()),
 			)
@@ -186,14 +188,9 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 		// a deduplicated list of names for which to obtain certs
 		// (only if cert management not disabled for this server)
 		if srv.AutoHTTPS.DisableCerts {
-			app.logger.Warn("skipping automated certificate management for server because it is disabled", zap.String("server_name", srvName))
+			logger.Warn("skipping automated certificate management for server because it is disabled", zap.String("server_name", srvName))
 		} else {
 			for d := range serverDomainSet {
-				// the implicit Tailscale manager module will get its own certs at run-time
-				if isTailscaleDomain(d) {
-					continue
-				}
-
 				if certmagic.SubjectQualifiesForCert(d) &&
 					!srv.AutoHTTPS.Skipped(d, srv.AutoHTTPS.SkipCerts) {
 					// if a certificate for this name is already loaded,
@@ -201,7 +198,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 					// supposed to ignore loaded certificates
 					if !srv.AutoHTTPS.IgnoreLoadedCerts &&
 						len(app.tlsApp.AllMatchingCertificates(d)) > 0 {
-						app.logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded",
+						logger.Info("skipping automatic certificate management because one or more matching certificates are already loaded",
 							zap.String("domain", d),
 							zap.String("server_name", srvName),
 						)
@@ -212,7 +209,7 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 					// can handle that, but as a courtesy, warn the user
 					if strings.Contains(d, "*") &&
 						strings.Count(strings.Trim(d, "."), ".") == 1 {
-						app.logger.Warn("most clients do not trust second-level wildcard certificates (*.tld)",
+						logger.Warn("most clients do not trust second-level wildcard certificates (*.tld)",
 							zap.String("domain", d))
 					}
 
@@ -228,11 +225,11 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 
 		// nothing left to do if auto redirects are disabled
 		if srv.AutoHTTPS.DisableRedir {
-			app.logger.Warn("automatic HTTP->HTTPS redirects are disabled", zap.String("server_name", srvName))
+			logger.Warn("automatic HTTP->HTTPS redirects are disabled", zap.String("server_name", srvName))
 			continue
 		}
 
-		app.logger.Info("enabling automatic HTTP->HTTPS redirects", zap.String("server_name", srvName))
+		logger.Info("enabling automatic HTTP->HTTPS redirects", zap.String("server_name", srvName))
 
 		// create HTTP->HTTPS redirects
 		for _, listenAddr := range srv.Listen {
@@ -272,12 +269,15 @@ func (app *App) automaticHTTPSPhase1(ctx caddy.Context, repl *caddy.Replacer) er
 	// we now have a list of all the unique names for which we need certs;
 	// turn the set into a slice so that phase 2 can use it
 	app.allCertDomains = make([]string, 0, len(uniqueDomainsForCerts))
-	var internal []string
+	var internal, tailscale []string
 uniqueDomainsLoop:
 	for d := range uniqueDomainsForCerts {
-		// whether or not there is already an automation policy for this
-		// name, we should add it to the list to manage a cert for it
-		app.allCertDomains = append(app.allCertDomains, d)
+		if !isTailscaleDomain(d) {
+			// whether or not there is already an automation policy for this
+			// name, we should add it to the list to manage a cert for it,
+			// unless it's a Tailscale domain, because we don't manage those
+			app.allCertDomains = append(app.allCertDomains, d)
+		}
 
 		// some names we've found might already have automation policies
 		// explicitly specified for them; we should exclude those from
@@ -295,13 +295,15 @@ uniqueDomainsLoop:
 
 		// if no automation policy exists for the name yet, we
 		// will associate it with an implicit one
-		if !certmagic.SubjectQualifiesForPublicCert(d) {
+		if isTailscaleDomain(d) {
+			tailscale = append(tailscale, d)
+		} else if !certmagic.SubjectQualifiesForPublicCert(d) {
 			internal = append(internal, d)
 		}
 	}
 
 	// ensure there is an automation policy to handle these certs
-	err := app.createAutomationPolicies(ctx, internal)
+	err := app.createAutomationPolicies(ctx, internal, tailscale)
 	if err != nil {
 		return err
 	}
@@ -424,6 +426,10 @@ redirServersLoop:
 		}
 	}
 
+	logger.Debug("adjusted config",
+		zap.Reflect("tls", app.tlsApp),
+		zap.Reflect("http", app))
+
 	return nil
 }
 
@@ -466,7 +472,7 @@ func (app *App) makeRedirRoute(redirToPort uint, matcherSet MatcherSet) Route {
 // automation policy exists, it will be shallow-copied and used as the
 // base for the new ones (this is important for preserving behavior the
 // user intends to be "defaults").
-func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []string) error {
+func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames, tailscaleNames []string) error {
 	// before we begin, loop through the existing automation policies
 	// and, for any ACMEIssuers we find, make sure they're filled in
 	// with default values that might be specified in our HTTP app; also
@@ -480,6 +486,22 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 		app.tlsApp.Automation = new(caddytls.AutomationConfig)
 	}
 	for _, ap := range app.tlsApp.Automation.Policies {
+		// on-demand policies can have the tailscale manager added implicitly
+		// if there's no explicit manager configured -- for convenience
+		if ap.OnDemand && len(ap.Managers) == 0 {
+			var ts caddytls.Tailscale
+			if err := ts.Provision(ctx); err != nil {
+				return err
+			}
+			ap.Managers = []certmagic.Manager{ts}
+
+			// must reprovision the automation policy so that the underlying
+			// CertMagic config knows about the updated Managers
+			if err := ap.Provision(app.tlsApp); err != nil {
+				return fmt.Errorf("re-provisioning automation policy: %v", err)
+			}
+		}
+
 		// set up default issuer -- honestly, this is only
 		// really necessary because the HTTP app is opinionated
 		// and has settings which could be inferred as new
@@ -501,22 +523,6 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 			}
 		}
 
-		// if no external managers were configured, enable
-		// implicit Tailscale support for convenience
-		if ap.Managers == nil {
-			ts, err := implicitTailscale(ctx)
-			if err != nil {
-				return err
-			}
-			ap.Managers = []certmagic.Manager{ts}
-
-			// must reprovision the automation policy so that the underlying
-			// CertMagic config knows about the updated Managers
-			if err := ap.Provision(app.tlsApp); err != nil {
-				return fmt.Errorf("re-provisioning automation policy: %v", err)
-			}
-		}
-
 		// while we're here, is this the catch-all/base policy?
 		if !foundBasePolicy && len(ap.SubjectsRaw) == 0 {
 			basePolicy = ap
@@ -529,15 +535,6 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 		basePolicy = new(caddytls.AutomationPolicy)
 	}
 
-	if basePolicy.Managers == nil {
-		// add implicit Tailscale integration, for harmless convenience
-		ts, err := implicitTailscale(ctx)
-		if err != nil {
-			return err
-		}
-		basePolicy.Managers = []certmagic.Manager{ts}
-	}
-
 	// if the basePolicy has an existing ACMEIssuer (particularly to
 	// include any type that embeds/wraps an ACMEIssuer), let's use it
 	// (I guess we just use the first one?), otherwise we'll make one
@@ -642,6 +639,27 @@ func (app *App) createAutomationPolicies(ctx caddy.Context, internalNames []stri
 		}
 	}
 
+	// tailscale names go in their own automation policies because
+	// they require on-demand TLS to be enabled, which we obviously
+	// can't enable for everything
+	if len(tailscaleNames) > 0 {
+		policyCopy := *basePolicy
+		newPolicy := &policyCopy
+
+		var ts caddytls.Tailscale
+		if err := ts.Provision(ctx); err != nil {
+			return err
+		}
+
+		newPolicy.SubjectsRaw = tailscaleNames
+		newPolicy.Issuers = nil
+		newPolicy.Managers = append(newPolicy.Managers, ts)
+		err := app.tlsApp.AddAutomationPolicy(newPolicy)
+		if err != nil {
+			return err
+		}
+	}
+
 	// we just changed a lot of stuff, so double-check that it's all good
 	err := app.tlsApp.Validate()
 	if err != nil {
@@ -720,13 +738,6 @@ func (app *App) automaticHTTPSPhase2() error {
 	return nil
 }
 
-// implicitTailscale returns a new and provisioned Tailscale module configured to be optional.
-func implicitTailscale(ctx caddy.Context) (caddytls.Tailscale, error) {
-	ts := caddytls.Tailscale{Optional: true}
-	err := ts.Provision(ctx)
-	return ts, err
-}
-
 func isTailscaleDomain(name string) bool {
 	return strings.HasSuffix(strings.ToLower(name), ".ts.net")
 }
diff --git a/modules/caddytls/automation.go b/modules/caddytls/automation.go
index 58ffe4cb5a1..16647626ca0 100644
--- a/modules/caddytls/automation.go
+++ b/modules/caddytls/automation.go
@@ -95,9 +95,11 @@ type AutomationPolicy struct {
 	// Modules that can get a custom certificate to use for any
 	// given TLS handshake at handshake-time. Custom certificates
 	// can be useful if another entity is managing certificates
-	// and Caddy need only get it and serve it.
+	// and Caddy need only get it and serve it. Specifying a Manager
+	// enables on-demand TLS, i.e. it has the side-effect of setting
+	// the on_demand parameter to `true`.
 	//
-	// TODO: This is an EXPERIMENTAL feature. It is subject to change or removal.
+	// TODO: This is an EXPERIMENTAL feature. Subject to change or removal.
 	ManagersRaw []json.RawMessage `json:"get_certificate,omitempty" caddy:"namespace=tls.get_certificate inline_key=via"`
 
 	// If true, certificates will be requested with MustStaple. Not all
@@ -233,15 +235,18 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 
 	// on-demand TLS
 	var ond *certmagic.OnDemandConfig
-	if ap.OnDemand {
+	if ap.OnDemand || len(ap.Managers) > 0 {
 		// ask endpoint is now required after a number of negligence cases causing abuse;
 		// but is still allowed for explicit subjects (non-wildcard, non-unbounded),
-		// and for the internal issuer since it doesn't cause ACME issuer pressure
+		// for the internal issuer since it doesn't cause ACME issuer pressure
 		if ap.isWildcardOrDefault() && !ap.onlyInternalIssuer() && (tlsApp.Automation == nil || tlsApp.Automation.OnDemand == nil || tlsApp.Automation.OnDemand.Ask == "") {
 			return fmt.Errorf("on-demand TLS cannot be enabled without an 'ask' endpoint to prevent abuse; please refer to documentation for details")
 		}
 		ond = &certmagic.OnDemandConfig{
 			DecisionFunc: func(name string) error {
+				if tlsApp.Automation == nil || tlsApp.Automation.OnDemand == nil {
+					return nil
+				}
 				if err := onDemandAskRequest(tlsApp.logger, tlsApp.Automation.OnDemand.Ask, name); err != nil {
 					// distinguish true errors from denials, because it's important to elevate actual errors
 					if errors.Is(err, errAskDenied) {
@@ -264,6 +269,7 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 				}
 				return nil
 			},
+			Managers: ap.Managers,
 		}
 	}
 
@@ -277,10 +283,9 @@ func (ap *AutomationPolicy) Provision(tlsApp *TLS) error {
 			DisableStapling:    ap.DisableOCSPStapling,
 			ResponderOverrides: ap.OCSPOverrides,
 		},
-		Storage:  storage,
-		Issuers:  issuers,
-		Managers: ap.Managers,
-		Logger:   tlsApp.logger,
+		Storage: storage,
+		Issuers: issuers,
+		Logger:  tlsApp.logger,
 	}
 	ap.magic = certmagic.New(tlsApp.certCache, template)
 
diff --git a/modules/caddytls/certmanagers.go b/modules/caddytls/certmanagers.go
index 1b701ab2653..23af19d44ac 100644
--- a/modules/caddytls/certmanagers.go
+++ b/modules/caddytls/certmanagers.go
@@ -23,14 +23,6 @@ func init() {
 
 // Tailscale is a module that can get certificates from the local Tailscale process.
 type Tailscale struct {
-	// If true, this module will operate in "best-effort" mode and
-	// ignore "soft" errors; i.e. try Tailscale, and if it doesn't connect
-	// or return a certificate, oh well. Failure to connect to Tailscale
-	// results in a no-op instead of an error. Intended for the use case
-	// where this module is added implicitly for convenience, even if
-	// Tailscale isn't necessarily running.
-	Optional bool `json:"optional,omitempty"`
-
 	logger *zap.Logger
 }
 
@@ -60,16 +52,11 @@ func (ts Tailscale) GetCertificate(ctx context.Context, hello *tls.ClientHelloIn
 
 // canHazCertificate returns true if Tailscale reports it can get a certificate for the given ClientHello.
 func (ts Tailscale) canHazCertificate(ctx context.Context, hello *tls.ClientHelloInfo) (bool, error) {
-	if ts.Optional && !strings.HasSuffix(strings.ToLower(hello.ServerName), tailscaleDomainAliasEnding) {
+	if !strings.HasSuffix(strings.ToLower(hello.ServerName), tailscaleDomainAliasEnding) {
 		return false, nil
 	}
 	status, err := tscert.GetStatus(ctx)
 	if err != nil {
-		if ts.Optional {
-			// ignore error if we don't expect/require it to work anyway, but log it for debugging
-			ts.logger.Debug("error getting tailscale status", zap.Error(err), zap.String("server_name", hello.ServerName))
-			return false, nil
-		}
 		return false, err
 	}
 	for _, domain := range status.CertDomains {


Test patch: