Repository: compose-spec/compose-go
Pull request #655 (2024-07-04 09:41:13)

load secrets/configs from environment while parsing compose model
As we load compose model with environment, resolve secrets and config values based on declared environment value. 
This allows a nested model loading (using `include`) to resolve such values based on configured `env_file`, not just top-level environment.
when marshaling model to yaml/json resolved content is hidden if it was set by environment, to keep definition consistency


closes https://github.com/docker/compose/issues/11952


Patch:
diff --git a/loader/environment.go b/loader/environment.go
index e6983bb..1906d2d 100644
--- a/loader/environment.go
+++ b/loader/environment.go
@@ -22,9 +22,14 @@ import (
 	"github.com/compose-spec/compose-go/v2/types"
 )
 
-// Will update the environment variables for the format {- VAR} (without interpolation)
-// This function should resolve context environment vars for include (passed in env_file)
-func resolveServicesEnvironment(dict map[string]any, config types.ConfigDetails) {
+// ResolveEnvironment update the environment variables for the format {- VAR} (without interpolation)
+func ResolveEnvironment(dict map[string]any, environment types.Mapping) {
+	resolveServicesEnvironment(dict, environment)
+	resolveSecretsEnvironment(dict, environment)
+	resolveConfigsEnvironment(dict, environment)
+}
+
+func resolveServicesEnvironment(dict map[string]any, environment types.Mapping) {
 	services, ok := dict["services"].(map[string]any)
 	if !ok {
 		return
@@ -45,7 +50,7 @@ func resolveServicesEnvironment(dict map[string]any, config types.ConfigDetails)
 			if !ok {
 				continue
 			}
-			if found, ok := config.Environment[varEnv]; ok {
+			if found, ok := environment[varEnv]; ok {
 				envs = append(envs, fmt.Sprintf("%s=%s", varEnv, found))
 			} else {
 				// either does not exist or it was already resolved in interpolation
@@ -57,3 +62,49 @@ func resolveServicesEnvironment(dict map[string]any, config types.ConfigDetails)
 	}
 	dict["services"] = services
 }
+
+func resolveSecretsEnvironment(dict map[string]any, environment types.Mapping) {
+	secrets, ok := dict["secrets"].(map[string]any)
+	if !ok {
+		return
+	}
+
+	for name, cfg := range secrets {
+		secret, ok := cfg.(map[string]any)
+		if !ok {
+			continue
+		}
+		env, ok := secret["environment"].(string)
+		if !ok {
+			continue
+		}
+		if found, ok := environment[env]; ok {
+			secret["content"] = found
+		}
+		secrets[name] = secret
+	}
+	dict["secrets"] = secrets
+}
+
+func resolveConfigsEnvironment(dict map[string]any, environment types.Mapping) {
+	configs, ok := dict["configs"].(map[string]any)
+	if !ok {
+		return
+	}
+
+	for name, cfg := range configs {
+		config, ok := cfg.(map[string]any)
+		if !ok {
+			continue
+		}
+		env, ok := config["environment"].(string)
+		if !ok {
+			continue
+		}
+		if found, ok := environment[env]; ok {
+			config["content"] = found
+		}
+		configs[name] = config
+	}
+	dict["configs"] = configs
+}
diff --git a/loader/loader.go b/loader/loader.go
index 394aec3..fc23159 100644
--- a/loader/loader.go
+++ b/loader/loader.go
@@ -407,7 +407,7 @@ func loadYamlModel(ctx context.Context, config types.ConfigDetails, opts *Option
 			return nil, err
 		}
 	}
-	resolveServicesEnvironment(dict, config)
+	ResolveEnvironment(dict, config.Environment)
 
 	return dict, nil
 }
diff --git a/types/types.go b/types/types.go
index 56af93e..b0f97bf 100644
--- a/types/types.go
+++ b/types/types.go
@@ -782,9 +782,41 @@ type ExtendsConfig struct {
 // SecretConfig for a secret
 type SecretConfig FileObjectConfig
 
+// MarshalYAML makes SecretConfig implement yaml.Marshaller
+func (s SecretConfig) MarshalYAML() (interface{}, error) {
+	// secret content is set while loading model. Never marshall it
+	s.Content = ""
+	return FileObjectConfig(s), nil
+}
+
+// MarshalJSON makes SecretConfig implement json.Marshaller
+func (s SecretConfig) MarshalJSON() ([]byte, error) {
+	// secret content is set while loading model. Never marshall it
+	s.Content = ""
+	return json.Marshal(FileObjectConfig(s))
+}
+
 // ConfigObjConfig is the config for the swarm "Config" object
 type ConfigObjConfig FileObjectConfig
 
+// MarshalYAML makes ConfigObjConfig implement yaml.Marshaller
+func (s ConfigObjConfig) MarshalYAML() (interface{}, error) {
+	// config content may have been set from environment while loading model. Marshall actual source
+	if s.Environment != "" {
+		s.Content = ""
+	}
+	return FileObjectConfig(s), nil
+}
+
+// MarshalJSON makes ConfigObjConfig implement json.Marshaller
+func (s ConfigObjConfig) MarshalJSON() ([]byte, error) {
+	// config content may have been set from environment while loading model. Marshall actual source
+	if s.Environment != "" {
+		s.Content = ""
+	}
+	return json.Marshal(FileObjectConfig(s))
+}
+
 type IncludeConfig struct {
 	Path             StringList `yaml:"path,omitempty" json:"path,omitempty"`
 	ProjectDirectory string     `yaml:"project_directory,omitempty" json:"project_directory,omitempty"`


Test patch:
diff --git a/loader/full-struct_test.go b/loader/full-struct_test.go
index a346b90..9d12274 100644
--- a/loader/full-struct_test.go
+++ b/loader/full-struct_test.go
@@ -588,6 +588,7 @@ func secrets(workingDir string) map[string]types.SecretConfig {
 		"secret4": {
 			Name:        "bar",
 			Environment: "BAR",
+			Content:     "this is a secret",
 			Extensions: map[string]interface{}{
 				"x-bar": "baz",
 				"x-foo": "bar",
diff --git a/loader/loader_test.go b/loader/loader_test.go
index d2d7ee2..5c93e69 100644
--- a/loader/loader_test.go
+++ b/loader/loader_test.go
@@ -3288,3 +3288,29 @@ services:
 	build := p.Services["test"].Build
 	assert.DeepEqual(t, build.Entitlements, []string{"network.host", "security.insecure"})
 }
+
+func TestLoadSecretEnvironment(t *testing.T) {
+	config, err := loadYAMLWithEnv(`
+name: load-secret-environment
+configs:
+  config:
+    environment: GA
+secrets:
+  secret:
+    environment: MEU
+`, map[string]string{
+		"GA":  "BU",
+		"MEU": "Shadoks",
+	})
+	assert.NilError(t, err)
+	assert.DeepEqual(t, config.Configs, types.Configs{
+		"config": {
+			Environment: "GA",
+			Content:     "BU",
+		}})
+	assert.DeepEqual(t, config.Secrets, types.Secrets{
+		"secret": {
+			Environment: "MEU",
+			Content:     "Shadoks",
+		}})
+}