Repository: compose-spec/compose-go
Pull request #444 (2023-08-16 17:58:26)

load: include details about included files on `Project`
Add a map to `Project` that has Compose YAML filename as key and any `IncludeConfig`s loaded from it.

This is populated as the project is recursively loaded.

For example:
```
proj/
  compose.yaml
  a/
    compose.yaml
  b/
    compose.yaml
```

If `project/compose.yaml` has:
```yaml
include: ['./a/compose.yaml']
```

And `project/a/compose.yaml` has:
```yaml
include: ['../b/compose.yaml']
```

The final result after load is conceptually:
```json
{
  "proj/compose.yaml": ["proj/a/compose.yaml"],
  "proj/a/compose.yaml": ["proj/b/compose.yaml"],
}
```
(Note: in reality, it's a list of `IncludeConfig`, which has multiple fields. Example is simplified.)

Relative path resolution is based on overall loader configuration, but note that disabling it does not work properly for `include` currently due to other issues.

This makes it possible for the caller to understand a bit more about the loaded project resources. We're really overdue for a bit of an overhaul/refactor of the loader - at that point, I think it might be better to have a `Loader` object type that can track stuff like this on the instance because it's weirdly both part of the project and NOT part of the project at the moment (similar to `Profiles`, project name, etc).


Patch:
diff --git a/loader/include.go b/loader/include.go
index 026dcf9..7a736aa 100644
--- a/loader/include.go
+++ b/loader/include.go
@@ -45,14 +45,17 @@ var transformIncludeConfig TransformerFunc = func(data interface{}) (interface{}
 	}
 }
 
-func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) {
+func loadInclude(ctx context.Context, filename string, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, map[string][]types.IncludeConfig, error) {
+	included := make(map[string][]types.IncludeConfig)
 	for _, r := range model.Include {
+		included[filename] = append(included[filename], r)
+
 		for i, p := range r.Path {
 			for _, loader := range options.ResourceLoaders {
 				if loader.Accept(p) {
 					path, err := loader.Load(ctx, p)
 					if err != nil {
-						return nil, err
+						return nil, nil, err
 					}
 					p = path
 					break
@@ -72,7 +75,7 @@ func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *
 
 		env, err := dotenv.GetEnvFromFile(configDetails.Environment, r.ProjectDirectory, r.EnvFile)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		config := types.ConfigDetails{
@@ -87,16 +90,19 @@ func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *
 		}
 		imported, err := load(ctx, config, loadOptions, loaded)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
+		}
+		for k, v := range imported.IncludeReferences {
+			included[k] = append(included[k], v...)
 		}
 
 		err = importResources(model, imported, r.Path)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 	model.Include = nil
-	return model, nil
+	return model, included, nil
 }
 
 // importResources import into model all resources defined by imported, and report error on conflict
diff --git a/loader/loader.go b/loader/loader.go
index b6f8fe9..6b80ab2 100644
--- a/loader/loader.go
+++ b/loader/loader.go
@@ -250,6 +250,7 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 	}
 	loaded = append(loaded, mainFile)
 
+	includeRefs := make(map[string][]types.IncludeConfig)
 	for i, file := range configDetails.ConfigFiles {
 		var postProcessor PostProcessor
 		configDict := file.Config
@@ -285,10 +286,14 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 		}
 
 		if !opts.SkipInclude {
-			cfg, err = loadInclude(ctx, configDetails, cfg, opts, loaded)
+			var included map[string][]types.IncludeConfig
+			cfg, included, err = loadInclude(ctx, file.Filename, configDetails, cfg, opts, loaded)
 			if err != nil {
 				return nil, err
 			}
+			for k, v := range included {
+				includeRefs[k] = append(includeRefs[k], v...)
+			}
 		}
 
 		if i == 0 {
@@ -321,6 +326,10 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options,
 		Extensions:  model.Extensions,
 	}
 
+	if len(includeRefs) != 0 {
+		project.IncludeReferences = includeRefs
+	}
+
 	if !opts.SkipNormalization {
 		err := Normalize(project)
 		if err != nil {
diff --git a/loader/paths.go b/loader/paths.go
index 45bbc54..61e79f0 100644
--- a/loader/paths.go
+++ b/loader/paths.go
@@ -64,6 +64,25 @@ func ResolveRelativePaths(project *types.Project) error {
 			project.Volumes[name] = config
 		}
 	}
+
+	// don't coerce a nil map to an empty map
+	if project.IncludeReferences != nil {
+		absIncludes := make(map[string][]types.IncludeConfig, len(project.IncludeReferences))
+		for filename, config := range project.IncludeReferences {
+			filename = absPath(project.WorkingDir, filename)
+			absConfigs := make([]types.IncludeConfig, len(config))
+			for i, c := range config {
+				absConfigs[i] = types.IncludeConfig{
+					Path:             resolvePaths(project.WorkingDir, c.Path),
+					ProjectDirectory: absPath(project.WorkingDir, c.ProjectDirectory),
+					EnvFile:          resolvePaths(project.WorkingDir, c.EnvFile),
+				}
+			}
+			absIncludes[filename] = absConfigs
+		}
+		project.IncludeReferences = absIncludes
+	}
+
 	return nil
 }
 
@@ -133,3 +152,14 @@ func isRemoteContext(maybeURL string) bool {
 	}
 	return false
 }
+
+func resolvePaths(basePath string, in types.StringList) types.StringList {
+	if in == nil {
+		return nil
+	}
+	ret := make(types.StringList, len(in))
+	for i := range in {
+		ret[i] = absPath(basePath, in[i])
+	}
+	return ret
+}
diff --git a/types/project.go b/types/project.go
index 59b6b2b..2a82ccf 100644
--- a/types/project.go
+++ b/types/project.go
@@ -36,16 +36,22 @@ import (
 
 // Project is the result of loading a set of compose files
 type Project struct {
-	Name         string     `yaml:"name,omitempty" json:"name,omitempty"`
-	WorkingDir   string     `yaml:"-" json:"-"`
-	Services     Services   `yaml:"services" json:"services"`
-	Networks     Networks   `yaml:"networks,omitempty" json:"networks,omitempty"`
-	Volumes      Volumes    `yaml:"volumes,omitempty" json:"volumes,omitempty"`
-	Secrets      Secrets    `yaml:"secrets,omitempty" json:"secrets,omitempty"`
-	Configs      Configs    `yaml:"configs,omitempty" json:"configs,omitempty"`
-	Extensions   Extensions `yaml:"#extensions,inline" json:"-"` // https://github.com/golang/go/issues/6213
-	ComposeFiles []string   `yaml:"-" json:"-"`
-	Environment  Mapping    `yaml:"-" json:"-"`
+	Name       string     `yaml:"name,omitempty" json:"name,omitempty"`
+	WorkingDir string     `yaml:"-" json:"-"`
+	Services   Services   `yaml:"services" json:"services"`
+	Networks   Networks   `yaml:"networks,omitempty" json:"networks,omitempty"`
+	Volumes    Volumes    `yaml:"volumes,omitempty" json:"volumes,omitempty"`
+	Secrets    Secrets    `yaml:"secrets,omitempty" json:"secrets,omitempty"`
+	Configs    Configs    `yaml:"configs,omitempty" json:"configs,omitempty"`
+	Extensions Extensions `yaml:"#extensions,inline" json:"-"` // https://github.com/golang/go/issues/6213
+
+	// IncludeReferences is keyed by Compose YAML filename and contains config for
+	// other Compose YAML files it directly triggered a load of via `include`.
+	//
+	// Note: this is
+	IncludeReferences map[string][]IncludeConfig `yaml:"-" json:"-"`
+	ComposeFiles      []string                   `yaml:"-" json:"-"`
+	Environment       Mapping                    `yaml:"-" json:"-"`
 
 	// DisabledServices track services which have been disable as profile is not active
 	DisabledServices Services `yaml:"-" json:"-"`
diff --git a/types/types.go b/types/types.go
index a9a515f..517d9a4 100644
--- a/types/types.go
+++ b/types/types.go
@@ -491,6 +491,16 @@ func NewMapping(values []string) Mapping {
 	return mapping
 }
 
+// convert values into a set of KEY=VALUE strings
+func (m Mapping) Values() []string {
+	values := make([]string, 0, len(m))
+	for k, v := range m {
+		values = append(values, fmt.Sprintf("%s=%s", k, v))
+	}
+	sort.Strings(values)
+	return values
+}
+
 // ToMappingWithEquals converts Mapping into a MappingWithEquals with pointer references
 func (m Mapping) ToMappingWithEquals() MappingWithEquals {
 	mapping := MappingWithEquals{}


Test patch:
diff --git a/loader/loader_test.go b/loader/loader_test.go
index 317b89b..130c778 100644
--- a/loader/loader_test.go
+++ b/loader/loader_test.go
@@ -2545,6 +2545,15 @@ services:
 			},
 		},
 	})
+	assert.DeepEqual(t, p.IncludeReferences, map[string][]types.IncludeConfig{
+		filepath.Join(workingDir, "filename0.yml"): {
+			{
+				Path:             []string{filepath.Join(workingDir, "testdata", "subdir", "compose-test-extends-imported.yaml")},
+				ProjectDirectory: workingDir,
+				EnvFile:          []string{filepath.Join(workingDir, "testdata", "subdir", "extra.env")},
+			},
+		},
+	})
 	assert.NilError(t, err)
 }
 
diff --git a/types/types_test.go b/types/types_test.go
index e4e7a13..495b8fe 100644
--- a/types/types_test.go
+++ b/types/types_test.go
@@ -368,3 +368,14 @@ func equalTrimSpace(x interface{}, y interface{}) is.Comparison {
 	}
 	return is.DeepEqual(trim(x), trim(y))
 }
+
+func TestMappingValues(t *testing.T) {
+	values := []string{"BAR=QIX", "FOO=BAR", "QIX=ZOT"}
+	mapping := NewMapping(values)
+	assert.DeepEqual(t, mapping, Mapping{
+		"FOO": "BAR",
+		"BAR": "QIX",
+		"QIX": "ZOT",
+	})
+	assert.DeepEqual(t, mapping.Values(), values)
+}