Repository: lucaslorentz/caddy-docker-proxy
Pull request #480 (2023-04-25 16:58:22)

Per container ingress network override
This PR adds support for overriding the ingress network for a container via a `caddy_ingress_network` label. Relates to #157

~Pushing this up for any early feedback - the code compiles, but I still need to test & add docs and tests are ready.~
Should be ready to go.

lucaslorentz: Hey, thanks @kiwiz for working on that!

The place you started extending is called only once when caddy is initialized and it will not update if new containers are created with new networks. It will also share the ingress network with all containers, instead of using it only for the container that declares the label.

My recommendation is to do it in https://github.com/lucaslorentz/caddy-docker-proxy/blob/master/generator/containers.go#L17 and https://github.com/lucaslorentz/caddy-docker-proxy/blob/master/generator/services.go#L52

### For containers.go

I did a docker inspect in a container and found the network name is in the key of the network map, as per the image below.
![Screenshot 2023-04-25 at 19 45 02](https://user-images.githubusercontent.com/2268284/234359785-ac458b00-8a34-4fb2-be00-b7b18483b82c.png)

I think this will hold true for docker, not sure if podman will be like that, but it should be fine for now.

So, I think changing this block:
```
	for _, network := range container.NetworkSettings.Networks {
		if !ingress || g.ingressNetworks[network.NetworkID] {
			ips = append(ips, network.IPAddress)
		}
	}
```

Into something like:
```
	ingressNetworkFromLabel := container.Labels[IngressNetworkLabel]
	for networkName, network := range container.NetworkSettings.Networks {
		if !ingress || g.ingressNetworks[network.NetworkID] || networkName == ingressNetworkFromLabel {
			ips = append(ips, network.IPAddress)
		}
	}
```
Might be enough, but I haven't tested.

### For services.go

I didn't sketch any code for this one, but I did inspect to check where the network name is:
![Screenshot 2023-04-25 at 19 44 18](https://user-images.githubusercontent.com/2268284/234360123-4ab01e5d-99e1-4054-a642-8222c85176b7.png)

We could write similar code for this as well.

If you're not using docker swarm, feel free to implement it only for containers, I don't mind merging it without support for services for now.




Patch:
diff --git a/README.md b/README.md
index 1d9e1e6..90f6723 100644
--- a/README.md
+++ b/README.md
@@ -330,7 +330,7 @@ For containers, that would be the container IPs.
 
 Only containers/services that are connected to Caddy ingress networks are used.
 
-:warning: caddy docker proxy does a best effort to automatically detect what are the ingress networks. But that logic fails on some scenarios: [#207](https://github.com/lucaslorentz/caddy-docker-proxy/issues/207). To have a more resilient solution, you can manually configure Caddy ingress network using CLI option `ingress-networks` or environment variable `CADDY_INGRESS_NETWORKS`.
+:warning: caddy docker proxy does a best effort to automatically detect what are the ingress networks. But that logic fails on some scenarios: [#207](https://github.com/lucaslorentz/caddy-docker-proxy/issues/207). To have a more resilient solution, you can manually configure Caddy ingress network using CLI option `ingress-networks`, environment variable `CADDY_INGRESS_NETWORKS` or label `caddy_ingress_network`.
 
 Usage: `upstreams [http|https] [port]`  
 
diff --git a/generator/containers.go b/generator/containers.go
index be23e54..2b12f3e 100644
--- a/generator/containers.go
+++ b/generator/containers.go
@@ -14,11 +14,23 @@ func (g *CaddyfileGenerator) getContainerCaddyfile(container *types.Container, l
 	})
 }
 
-func (g *CaddyfileGenerator) getContainerIPAddresses(container *types.Container, logger *zap.Logger, ingress bool) ([]string, error) {
+func (g *CaddyfileGenerator) getContainerIPAddresses(container *types.Container, logger *zap.Logger, onlyIngressIps bool) ([]string, error) {
 	ips := []string{}
 
-	for _, network := range container.NetworkSettings.Networks {
-		if !ingress || g.ingressNetworks[network.NetworkID] {
+	ingressNetworkFromLabel, overrideNetwork := container.Labels[IngressNetworkLabel]
+
+	for networkName, network := range container.NetworkSettings.Networks {
+		include := false
+
+		if !onlyIngressIps  {
+			include = true
+		} else if overrideNetwork {
+			include = networkName == ingressNetworkFromLabel
+		} else {
+			include = g.ingressNetworks[network.NetworkID]
+		}
+
+		if include {
 			ips = append(ips, network.IPAddress)
 		}
 	}
diff --git a/generator/generator.go b/generator/generator.go
index 6a5f5c2..b7f9f51 100644
--- a/generator/generator.go
+++ b/generator/generator.go
@@ -21,6 +21,8 @@ import (
 // DefaultLabelPrefix for caddy labels in docker
 const DefaultLabelPrefix = "caddy"
 
+const IngressNetworkLabel = "caddy_ingress_network"
+
 const swarmAvailabilityCacheInterval = 1 * time.Minute
 
 // CaddyfileGenerator generates caddyfile from docker configuration
diff --git a/generator/services.go b/generator/services.go
index 116ce5c..3cbc6ce 100644
--- a/generator/services.go
+++ b/generator/services.go
@@ -20,12 +20,12 @@ func (g *CaddyfileGenerator) getServiceCaddyfile(service *swarm.Service, logger
 	})
 }
 
-func (g *CaddyfileGenerator) getServiceProxyTargets(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) {
+func (g *CaddyfileGenerator) getServiceProxyTargets(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) {
 	if g.options.ProxyServiceTasks {
-		return g.getServiceTasksIps(service, logger, ingress)
+		return g.getServiceTasksIps(service, logger, onlyIngressIps)
 	}
 
-	_, err := g.getServiceVirtualIps(service, logger, ingress)
+	_, err := g.getServiceVirtualIps(service, logger, onlyIngressIps)
 	if err != nil {
 		return nil, err
 	}
@@ -33,11 +33,11 @@ func (g *CaddyfileGenerator) getServiceProxyTargets(service *swarm.Service, logg
 	return []string{service.Spec.Name}, nil
 }
 
-func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) {
+func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) {
 	virtualIps := []string{}
 
 	for _, virtualIP := range service.Endpoint.VirtualIPs {
-		if !ingress || g.ingressNetworks[virtualIP.NetworkID] {
+		if !onlyIngressIps || g.ingressNetworks[virtualIP.NetworkID] {
 			virtualIps = append(virtualIps, virtualIP.Addr)
 		}
 	}
@@ -49,7 +49,7 @@ func (g *CaddyfileGenerator) getServiceVirtualIps(service *swarm.Service, logger
 	return virtualIps, nil
 }
 
-func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger *zap.Logger, ingress bool) ([]string, error) {
+func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger *zap.Logger, onlyIngressIps bool) ([]string, error) {
 	taskListFilter := filters.NewArgs()
 	taskListFilter.Add("service", service.ID)
 	taskListFilter.Add("desired-state", "running")
@@ -66,8 +66,20 @@ func (g *CaddyfileGenerator) getServiceTasksIps(service *swarm.Service, logger *
 		for _, task := range tasks {
 			if task.Status.State == swarm.TaskStateRunning {
 				hasRunningTasks = true
+				ingressNetworkFromLabel, overrideNetwork := service.Spec.Labels[IngressNetworkLabel]
+
 				for _, networkAttachment := range task.NetworksAttachments {
-					if !ingress || g.ingressNetworks[networkAttachment.Network.ID] {
+					include := false
+
+					if !onlyIngressIps {
+						include = true
+					} else if overrideNetwork {
+						include = networkAttachment.Network.Spec.Name == ingressNetworkFromLabel
+					} else {
+						include = g.ingressNetworks[networkAttachment.Network.ID]
+					}
+
+					if include {
 						for _, address := range networkAttachment.Addresses {
 							ipAddress, _, _ := net.ParseCIDR(address)
 							tasksIps = append(tasksIps, ipAddress.String())


Test patch:
diff --git a/generator/containers_test.go b/generator/containers_test.go
index 15707d4..4707992 100644
--- a/generator/containers_test.go
+++ b/generator/containers_test.go
@@ -138,6 +138,52 @@ func TestContainers_ManualIngressNetworks(t *testing.T) {
 	}, expectedCaddyfile, expectedLogs)
 }
 
+func TestContainers_OverrideIngressNetworks(t *testing.T) {
+	dockerClient := createBasicDockerClientMock()
+	dockerClient.NetworksData = []types.NetworkResource{
+		{
+			ID:   "other-network-id",
+			Name: "other-network-name",
+		},
+		{
+			ID:   "another-network-id",
+			Name: "another-network-name",
+		},
+	}
+	dockerClient.ContainersData = []types.Container{
+		{
+			ID: "CONTAINER-ID",
+			NetworkSettings: &types.SummaryNetworkSettings{
+				Networks: map[string]*network.EndpointSettings{
+					"other-network": {
+						IPAddress: "10.0.0.1",
+						NetworkID: "other-network-id",
+					},
+					"another-network": {
+						IPAddress: "10.0.0.2",
+						NetworkID: "other-network-id",
+					},
+				},
+			},
+			Labels: map[string]string{
+				"caddy_ingress_network":      "another-network",
+				fmtLabel("%s"):               "service.testdomain.com",
+				fmtLabel("%s.reverse_proxy"): "{{upstreams}}",
+			},
+		},
+	}
+
+	const expectedCaddyfile = "service.testdomain.com {\n" +
+		"	reverse_proxy 10.0.0.2\n" +
+		"}\n"
+
+	const expectedLogs = otherIngressNetworksMapLog + swarmIsAvailableLog
+
+	testGeneration(t, dockerClient, func(options *config.Options) {
+		options.IngressNetworks = []string{"other-network-name"}
+	}, expectedCaddyfile, expectedLogs)
+}
+
 func TestContainers_Replicas(t *testing.T) {
 	dockerClient := createBasicDockerClientMock()
 	dockerClient.ContainersData = []types.Container{
diff --git a/generator/services_test.go b/generator/services_test.go
index 458a004..06fdec5 100644
--- a/generator/services_test.go
+++ b/generator/services_test.go
@@ -382,6 +382,84 @@ func TestServiceTasks_ManualIngressNetwork(t *testing.T) {
 	}, expectedCaddyfile, expectedLogs)
 }
 
+func TestServiceTasks_OverrideIngressNetwork(t *testing.T) {
+	dockerClient := createBasicDockerClientMock()
+	dockerClient.ServicesData = []swarm.Service{
+		{
+			ID: "SERVICEID",
+			Spec: swarm.ServiceSpec{
+				Annotations: swarm.Annotations{
+					Name: "service",
+					Labels: map[string]string{
+						"caddy_ingress_network":      "another-network",
+						fmtLabel("%s"):               "service.testdomain.com",
+						fmtLabel("%s.reverse_proxy"): "{{upstreams 5000}}",
+					},
+				},
+			},
+			Endpoint: swarm.Endpoint{
+				VirtualIPs: []swarm.EndpointVirtualIP{
+					{
+						NetworkID: caddyNetworkID,
+					},
+				},
+			},
+		},
+	}
+	dockerClient.NetworksData = []types.NetworkResource{
+		{
+			ID:   "other-network-id",
+			Name: "other-network-name",
+		},
+		{
+			ID:   "another-network-id",
+			Name: "another-network-name",
+		},
+	}
+	dockerClient.TasksData = []swarm.Task{
+		{
+			ServiceID: "SERVICEID",
+			NetworksAttachments: []swarm.NetworkAttachment{
+				{
+					Network: swarm.Network{
+						ID: "other-network-id",
+						Spec: swarm.NetworkSpec{
+							Annotations: swarm.Annotations{
+								Name: "other-network",
+							},
+						},
+					},
+					Addresses: []string{"10.0.0.1/24"},
+				},
+				{
+					Network: swarm.Network{
+						ID: "another-network-id",
+						Spec: swarm.NetworkSpec{
+							Annotations: swarm.Annotations{
+								Name: "another-network",
+							},
+						},
+					},
+					Addresses: []string{"10.0.0.2/24"},
+				},
+			},
+			DesiredState: swarm.TaskStateRunning,
+			Status:       swarm.TaskStatus{State: swarm.TaskStateRunning},
+		},
+	}
+
+	const expectedCaddyfile = "service.testdomain.com {\n" +
+		"	reverse_proxy 10.0.0.2:5000\n" +
+		"}\n"
+
+	const expectedLogs = otherIngressNetworksMapLog + swarmIsAvailableLog
+
+	testGeneration(t, dockerClient, func(options *config.Options) {
+		options.ProxyServiceTasks = true
+		options.IngressNetworks = []string{"other-network-name"}
+	}, expectedCaddyfile, expectedLogs)
+}
+
 func TestServiceTasks_Running(t *testing.T) {
 	dockerClient := createBasicDockerClientMock()
 	dockerClient.ServicesData = []swarm.Service{
diff --git a/tests/functions.sh b/tests/functions.sh
index 436da7f..bf6e894 100644
--- a/tests/functions.sh
+++ b/tests/functions.sh
@@ -2,8 +2,8 @@
 
 function retry {
   local n=0
-  local max=5
-  local delay=20
+  local max=20
+  local delay=5
   while true; do
     ((n=n+1))
     "$@" && break || {
diff --git a/tests/ingress-networks/compose.yaml b/tests/ingress-networks/compose.yaml
index 617a3ef..bf3a1a4 100644
--- a/tests/ingress-networks/compose.yaml
+++ b/tests/ingress-networks/compose.yaml
@@ -11,6 +11,7 @@ services:
       - controller
       - ingress_0
       - ingress_1
+      - ingress_2
     environment:
       - CADDY_DOCKER_MODE=server
       - CADDY_CONTROLLER_NETWORK=10.200.200.0/24
@@ -54,11 +55,29 @@ services:
         caddy.reverse_proxy: "{{upstreams 80}}"
         caddy.tls: "internal"
 
+  # Proxy to service
+  whoami2:
+    image: containous/whoami
+    networks:
+      - internal
+      - ingress_2
+    deploy:
+      labels:
+        caddy: whoami2.example.com
+        caddy.reverse_proxy: "{{upstreams 80}}"
+        caddy.tls: "internal"
+        caddy_ingress_network: ingress_2
+
 networks:
   ingress_0:
     name: ingress_0
   ingress_1:
     name: ingress_1
+  ingress_2:
+    name: ingress_2
+  internal:
+    name: internal
+    internal: true
   controller:
     driver: overlay
     ipam:
diff --git a/tests/ingress-networks/run.sh b/tests/ingress-networks/run.sh
index 65209f5..fc45387 100644
--- a/tests/ingress-networks/run.sh
+++ b/tests/ingress-networks/run.sh
@@ -7,7 +7,8 @@ set -e
 docker stack deploy -c compose.yaml --prune caddy_test
 
 retry curl --show-error -s -k -f --resolve whoami0.example.com:443:127.0.0.1 https://whoami0.example.com &&
-retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com || {
+retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com &&
+retry curl --show-error -s -k -f --resolve whoami2.example.com:443:127.0.0.1 https://whoami2.example.com || {
     docker service logs caddy_test_caddy_controller
     docker service logs caddy_test_caddy_server
     exit 1
diff --git a/tests/standalone/compose.yaml b/tests/standalone/compose.yaml
index 6bb3bd0..a78c071 100644
--- a/tests/standalone/compose.yaml
+++ b/tests/standalone/compose.yaml
@@ -56,6 +56,18 @@ services:
       caddy.reverse_proxy: "{{upstreams 80}}"
       caddy.tls: "internal"
 
+  # Proxy to container
+  whoami4:
+    image: containous/whoami
+    networks:
+      - internal
+      - caddy
+    labels:
+      caddy: whoami4.example.com
+      caddy.reverse_proxy: "{{upstreams 80}}"
+      caddy.tls: "internal"
+      caddy_ingress_network: caddy_test
+
   # Proxy with matches and route
   echo_0:
     image: containous/whoami
@@ -74,4 +86,7 @@ services:
 networks:
   caddy:
     name: caddy_test
-    external: true
\ No newline at end of file
+    external: true
+  internal:
+    name: internal
+    internal: true
diff --git a/tests/standalone/run.sh b/tests/standalone/run.sh
index 71a8ea8..56c5193 100755
--- a/tests/standalone/run.sh
+++ b/tests/standalone/run.sh
@@ -10,6 +10,7 @@ retry curl --show-error -s -k -f --resolve whoami0.example.com:443:127.0.0.1 htt
 retry curl --show-error -s -k -f --resolve whoami1.example.com:443:127.0.0.1 https://whoami1.example.com &&
 retry curl --show-error -s -k -f --resolve whoami2.example.com:443:127.0.0.1 https://whoami2.example.com &&
 retry curl --show-error -s -k -f --resolve whoami3.example.com:443:127.0.0.1 https://whoami3.example.com &&
+retry curl --show-error -s -k -f --resolve whoami4.example.com:443:127.0.0.1 https://whoami4.example.com &&
 retry curl --show-error -s -k -f --resolve echo0.example.com:443:127.0.0.1 https://echo0.example.com/sourcepath/something || {
     docker service logs caddy_test_caddy
     exit 1