Repository: rancher/fleet (go)

Issue:
Feature Request: Cluster specific registry that the agent image is pulled from
### Is your feature request related to a problem?

Yes. Rancher has the ability to specify cluster specific registries. These are useful in a variety of ways. When fleet is used with Rancher the cluster specific registry is not passed on for fleet to use. This is a problem when Rancher relies on fleet and it doesn't work with the cluster specific registry.

### Solution you'd like

1. To have the fleet Cluster CRD extended to add a registry
2. When fleet installs an agent in that cluster the registry on the Cluster CR is used for the image to install

### Alternatives you've considered

N/A

### Anything else?

This is a sub-task of the epic at [#712](https://github.com/rancher/fleet/issues/712)

This should not be closed until Rancher is updated to use the change. The change, I believe, needs to be made at [here](https://github.com/rancher/rancher/blob/3e5fb4a537731df7ed96cc9ddae6a3902a57c8b1/pkg/controllers/provisioningv2/fleetcluster/fleetcluster.go#L118-L129). I think you can use the function [`GetPrivateRepoURLFromCluster`](https://github.com/rancher/rancher/blob/3e5fb4a537731df7ed96cc9ddae6a3902a57c8b1/pkg/provisioningv2/image/resolve.go#L34) to get the private repo from the cluster instance.

Pull request:
Cluster specific registry replaces global registry
If both systemDefaultRegistry and privateRepoURL are present, remove systemDefaultRegistry from the image arg before prefixing with privateRepoURL.

To be able to remove the systemDefaultRegistry, we need to store it separately in the fleet config.

Fix #948

## Test

To test this pull request, you can run the following commands:

```shell
go test ./pkg/agent/...
```


Patch:
diff --git a/charts/fleet/templates/configmap.yaml b/charts/fleet/templates/configmap.yaml
index 6b8d6f05..07f1b592 100644
--- a/charts/fleet/templates/configmap.yaml
+++ b/charts/fleet/templates/configmap.yaml
@@ -5,6 +5,7 @@ metadata:
 data:
   config: |
     {
+      "systemDefaultRegistry": "{{ template "system_default_registry" . }}",
       "agentImage": "{{ template "system_default_registry" . }}{{.Values.agentImage.repository}}:{{.Values.agentImage.tag}}",
       "agentImagePullPolicy": "{{ .Values.agentImage.imagePullPolicy }}",
       "apiServerURL": "{{.Values.apiServerURL}}",
diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
index 5e6d8043..318d8e81 100644
--- a/pkg/agent/agent.go
+++ b/pkg/agent/agent.go
@@ -90,6 +90,7 @@ func AgentWithConfig(ctx context.Context, agentNamespace, controllerNamespace, a
 
 	mo := opts.ManifestOptions
 	mo.AgentImage = cfg.AgentImage
+	mo.SystemDefaultRegistry = cfg.SystemDefaultRegistry
 	mo.AgentImagePullPolicy = cfg.AgentImagePullPolicy
 	mo.CheckinInterval = cfg.AgentCheckinInternal.Duration.String()
 
diff --git a/pkg/agent/manifest.go b/pkg/agent/manifest.go
index 8067e79e..39b82815 100644
--- a/pkg/agent/manifest.go
+++ b/pkg/agent/manifest.go
@@ -26,12 +26,13 @@ const (
 )
 
 type ManifestOptions struct {
-	AgentEnvVars         []corev1.EnvVar
-	AgentImage           string
-	AgentImagePullPolicy string
-	CheckinInterval      string
-	Generation           string
-	PrivateRepoURL       string
+	AgentEnvVars          []corev1.EnvVar
+	AgentImage            string
+	AgentImagePullPolicy  string
+	CheckinInterval       string
+	Generation            string
+	PrivateRepoURL        string
+	SystemDefaultRegistry string
 }
 
 // Manifest builds and returns a deployment manifest for the fleet-agent with a
@@ -58,7 +59,7 @@ func Manifest(namespace string, agentScope string, opts ManifestOptions) []runti
 
 	// PrivateRepoURL = registry.yourdomain.com:5000
 	// DefaultAgentImage = "rancher/fleet-agent" + ":" + version.Version
-	image := resolve(opts.PrivateRepoURL, opts.AgentImage)
+	image := resolve(opts.SystemDefaultRegistry, opts.PrivateRepoURL, opts.AgentImage)
 
 	// if debug is enabled in controller, enable in agent too
 	debug := logrus.IsLevelEnabled(logrus.DebugLevel)
@@ -133,9 +134,12 @@ func Manifest(namespace string, agentScope string, opts ManifestOptions) []runti
 	return objs
 }
 
-func resolve(reg, image string) string {
-	if reg != "" && !strings.HasPrefix(image, reg) {
-		return path.Join(reg, image)
+func resolve(global, prefix, image string) string {
+	if global != "" && prefix != "" {
+		image = strings.TrimPrefix(image, global)
+	}
+	if prefix != "" && !strings.HasPrefix(image, prefix) {
+		return path.Join(prefix, image)
 	}
 
 	return image
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 16ae0874..6c965c75 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -37,6 +37,7 @@ var (
 type Config struct {
 	AgentImage                      string            `json:"agentImage,omitempty"`
 	AgentImagePullPolicy            string            `json:"agentImagePullPolicy,omitempty"`
+	SystemDefaultRegistry           string            `json:"systemDefaultRegistry,omitempty"`
 	AgentCheckinInternal            metav1.Duration   `json:"agentCheckinInternal,omitempty"`
 	ManageAgent                     *bool             `json:"manageAgent,omitempty"`
 	Labels                          map[string]string `json:"labels,omitempty"`
diff --git a/pkg/controllers/manageagent/manageagent.go b/pkg/controllers/manageagent/manageagent.go
index 5d71790a..ac8665d7 100644
--- a/pkg/controllers/manageagent/manageagent.go
+++ b/pkg/controllers/manageagent/manageagent.go
@@ -178,12 +178,13 @@ func (h *handler) newAgentBundle(ns string, cluster *fleet.Cluster) (runtime.Obj
 	objs := agent.Manifest(
 		agentNamespace, cluster.Spec.AgentNamespace,
 		agent.ManifestOptions{
-			AgentEnvVars:         cluster.Spec.AgentEnvVars,
-			AgentImage:           cfg.AgentImage,
-			AgentImagePullPolicy: cfg.AgentImagePullPolicy,
-			CheckinInterval:      cfg.AgentCheckinInternal.Duration.String(),
-			Generation:           "bundle",
-			PrivateRepoURL:       cluster.Spec.PrivateRepoURL,
+			AgentEnvVars:          cluster.Spec.AgentEnvVars,
+			AgentImage:            cfg.AgentImage,
+			AgentImagePullPolicy:  cfg.AgentImagePullPolicy,
+			CheckinInterval:       cfg.AgentCheckinInternal.Duration.String(),
+			Generation:            "bundle",
+			PrivateRepoURL:        cluster.Spec.PrivateRepoURL,
+			SystemDefaultRegistry: cfg.SystemDefaultRegistry,
 		},
 	)
 	agentYAML, err := yaml.Export(objs...)


Test patch:
diff --git a/pkg/agent/manifest_test.go b/pkg/agent/manifest_test.go
new file mode 100644
index 00000000..80329df8
--- /dev/null
+++ b/pkg/agent/manifest_test.go
@@ -0,0 +1,23 @@
+package agent
+
+import "testing"
+
+func TestImageResolve(t *testing.T) {
+	tests := []struct {
+		systemDefaultRegistry string
+		privateRepoURL        string
+		image                 string
+		expected              string
+	}{
+		{"", "", "rancher/fleet:dev", "rancher/fleet:dev"},
+		{"mirror.example/", "", "mirror.example/rancher/fleet:dev", "mirror.example/rancher/fleet:dev"},
+		{"mirror.example/", "local.example", "mirror.example/rancher/fleet:dev", "local.example/rancher/fleet:dev"},
+	}
+
+	for _, d := range tests {
+		image := resolve(d.systemDefaultRegistry, d.privateRepoURL, d.image)
+		if image != d.expected {
+			t.Errorf("expected %s, got %s", d.expected, image)
+		}
+	}
+}