Repository: score-spec/score-compose
Pull request #60 (2024-03-07 16:18:42)

feat: added resource state model, priming, and legacy output generation
This PR lays the ground work for the upcoming `score-compose generate` command which provides more formal resource provisioning.

In this PR we establish a state tree that contains final workload specs and the states of all provisioned resources. 

The flow works in the following steps:

1. load state from disk or construct new state in memory (this is what run does here)
2. add any new or updated score workloads into the state
3. "prime" resources. This generates initial resource states for all resources that haven't been provisioned before, it also updates metadata and params, and ensures that there are no conflicting definitions. Note: it does not remove old resources that are no longer declared in workloads.
4. ~"provision" resources and~ set outputs. We have not implemented any provisioning here! Instead we use the old logic that was in internal/compose/convert.go for setting legacy outputs from the resources we recognise.
5. "collect outputs" for each workload as input into the Compose conversion process. This ensures that `${resources.foo.blah}` knows what `foo` is, and has a function for looking up `blah`.
6. ~"persist" the state file~. This is skipped in the run command, for now.

NOTE that we don't go any further than this in this PR because it's already long enough, but the next steps are:

1. Build out the generate command which accepts multiple score files and persists the state tree to disk. Combine all the compose outputs together.
2. Define a provisioners interface and assign these to resources and provision them.

NOTE: there is no change in behaviour externally yet!

#### Types of changes
- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
- [X] New chore (expected functionality to be implemented)

#### Checklist:
- [ ] My change requires a change to the documentation.
  - [ ] I have updated the documentation accordingly.
- [X] I've signed off with an email address that matches the commit author.



Patch:
diff --git a/internal/command/init.go b/internal/command/init.go
index 416fb6b..12353c5 100644
--- a/internal/command/init.go
+++ b/internal/command/init.go
@@ -6,6 +6,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -79,27 +80,43 @@ acts as a namespace when multiple score files and containers are used.
 		initCmdScoreFile, _ := cmd.Flags().GetString("file")
 		initCmdComposeProject, _ := cmd.Flags().GetString("project")
 
+		// validate project
+		if initCmdComposeProject != "" {
+			cleanedInitCmdComposeProject := cleanComposeProjectName(initCmdComposeProject)
+			if cleanedInitCmdComposeProject != initCmdComposeProject {
+				return fmt.Errorf("invalid value for --project, it must match ^[a-z0-9][a-z0-9_-]*$")
+			}
+		}
+
 		sd, ok, err := project.LoadStateDirectory(".")
 		if err != nil {
 			return fmt.Errorf("failed to load existing state directory: %w", err)
 		} else if ok {
 			slog.Info(fmt.Sprintf("Found existing state directory '%s'", sd.Path))
-			if initCmdComposeProject != "" && sd.Config.ComposeProjectName != initCmdComposeProject {
-				sd.Config.ComposeProjectName = initCmdComposeProject
+			if initCmdComposeProject != "" && sd.State.ComposeProjectName != initCmdComposeProject {
+				sd.State.ComposeProjectName = initCmdComposeProject
 				if err := sd.Persist(); err != nil {
 					return fmt.Errorf("failed to persist new compose project name: %w", err)
 				}
 			}
 		} else {
+
 			slog.Info(fmt.Sprintf("Writing new state directory '%s'", project.DefaultRelativeStateDirectory))
 			wd, _ := os.Getwd()
 			sd := &project.StateDirectory{
-				Path:   project.DefaultRelativeStateDirectory,
-				Config: project.Config{ComposeProjectName: filepath.Base(wd)},
+				Path: project.DefaultRelativeStateDirectory,
+				State: project.State{
+					Workloads:          map[string]project.ScoreWorkloadState{},
+					Resources:          map[project.ResourceUid]project.ScoreResourceState{},
+					SharedState:        map[string]interface{}{},
+					ComposeProjectName: cleanComposeProjectName(filepath.Base(wd)),
+					MountsDirectory:    filepath.Join(project.DefaultRelativeStateDirectory, project.MountsDirectoryName),
+				},
 			}
 			if initCmdComposeProject != "" {
-				sd.Config.ComposeProjectName = initCmdComposeProject
+				sd.State.ComposeProjectName = initCmdComposeProject
 			}
+			slog.Info(fmt.Sprintf("Writing new state directory '%s' with project name '%s'", sd.Path, sd.State.ComposeProjectName))
 			if err := sd.Persist(); err != nil {
 				return fmt.Errorf("failed to persist new compose project name: %w", err)
 			}
@@ -130,3 +147,17 @@ func init() {
 
 	rootCmd.AddCommand(initCmd)
 }
+
+func cleanComposeProjectName(input string) string {
+	input = strings.ToLower(input)
+	isFirst := true
+	input = strings.Map(func(r rune) rune {
+		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || (!isFirst && ((r == '_') || (r == '-'))) {
+			isFirst = false
+			return r
+		}
+		isFirst = false
+		return -1
+	}, input)
+	return input
+}
diff --git a/internal/command/run.go b/internal/command/run.go
index c70ad1f..81bdc77 100644
--- a/internal/command/run.go
+++ b/internal/command/run.go
@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"maps"
 	"os"
 	"sort"
 	"strings"
@@ -27,6 +28,7 @@ import (
 	score "github.com/score-spec/score-go/types"
 
 	"github.com/score-spec/score-compose/internal/compose"
+	"github.com/score-spec/score-compose/internal/project"
 )
 
 const (
@@ -169,10 +171,37 @@ func run(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("validating workload spec: %w", err)
 	}
 
+	// Build a fake score-compose init state. We don't actually need to store or persist this because we're not doing
+	// anything iterative or stateful.
+	state := &project.State{MountsDirectory: "/dev/null"}
+	state, err = state.WithWorkload(&spec, &scoreFile)
+	if err != nil {
+		return fmt.Errorf("failed to add score file to state: %w", err)
+	}
+
+	// Prime the resources with initial state and validate any issues
+	state, err = state.WithPrimedResources()
+	if err != nil {
+		return fmt.Errorf("failed to prime resources: %w", err)
+	}
+
+	// Instead of actually calling the resource provisioning system, we skip it and fill in the supported resources
+	// ourselves.
+	vars := new(compose.EnvVarTracker)
+	state, err = fillInLegacyResourceOutputFunctions(spec.Metadata["name"].(string), state, vars)
+	if err != nil {
+		return fmt.Errorf("failed to provision resources: %w", err)
+	}
+
+	workloadResourceOutputs, err := state.GetResourceOutputForWorkload(spec.Metadata["name"].(string))
+	if err != nil {
+		return fmt.Errorf("failed to gather resource outputs: %w", err)
+	}
+
 	// Build docker-compose configuration
 	//
 	slog.Info("Building docker-compose configuration")
-	proj, vars, err := compose.ConvertSpec(&spec)
+	proj, err := compose.ConvertSpec(&spec, workloadResourceOutputs)
 	if err != nil {
 		return fmt.Errorf("building docker-compose configuration: %w", err)
 	}
@@ -239,3 +268,28 @@ func run(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func fillInLegacyResourceOutputFunctions(workloadName string, state *project.State, evt *compose.EnvVarTracker) (*project.State, error) {
+	out := *state
+	out.Resources = maps.Clone(state.Resources)
+	for resName, res := range state.Workloads[workloadName].Spec.Resources {
+		resUid := project.NewResourceUid(workloadName, resName, res.Type, res.Class, res.Id)
+		resState := state.Resources[resUid]
+		if resUid.Type() == "environment" {
+			if resUid.Class() != "default" {
+				return nil, fmt.Errorf("resources '%s': '%s.%s' is not supported in score-compose", resUid, resUid.Type(), resUid.Class())
+			}
+			resState.OutputLookupFunc = evt.LookupOutput
+		} else if resUid.Type() == "volume" && resUid.Class() == "default" {
+			resState.OutputLookupFunc = func(keys ...string) (interface{}, error) {
+				return nil, fmt.Errorf("resource has no outputs")
+			}
+		} else {
+			slog.Warn(fmt.Sprintf("resources.%s: '%s.%s' is not directly supported in score-compose, references will be converted to environment variables", resName, resUid.Type(), resUid.Class()))
+			fake := evt.GenerateResource(resName)
+			resState.OutputLookupFunc = fake.LookupOutput
+		}
+		out.Resources[resUid] = resState
+	}
+	return &out, nil
+}
diff --git a/internal/compose/convert.go b/internal/compose/convert.go
index 7d5f7e7..b755dc7 100644
--- a/internal/compose/convert.go
+++ b/internal/compose/convert.go
@@ -17,48 +17,28 @@ import (
 	compose "github.com/compose-spec/compose-go/v2/types"
 	score "github.com/score-spec/score-go/types"
 
+	"github.com/score-spec/score-compose/internal/project"
 	"github.com/score-spec/score-compose/internal/util"
 )
 
 // ConvertSpec converts SCORE specification into docker-compose configuration.
-func ConvertSpec(spec *score.Workload) (*compose.Project, *EnvVarTracker, error) {
+func ConvertSpec(spec *score.Workload, resources map[string]project.OutputLookupFunc) (*compose.Project, error) {
 	workloadName, ok := spec.Metadata["name"].(string)
 	if !ok || len(workloadName) == 0 {
-		return nil, nil, errors.New("workload metadata is missing a name")
+		return nil, errors.New("workload metadata is missing a name")
 	}
 
 	if len(spec.Containers) == 0 {
-		return nil, nil, errors.New("workload does not have any containers to convert into a compose service")
+		return nil, errors.New("workload does not have any containers to convert into a compose service")
 	}
 
 	var project = compose.Project{
 		Services: make(compose.Services),
 	}
 
-	// Track any uses of the environment resource or resources that are overridden with an env provider using the tracker.
-	envVarTracker := new(EnvVarTracker)
-	resources := make(map[string]ResourceWithOutputs)
-	// The first thing we must do is validate or create the resources this workload depends on.
-	// NOTE: this will soon be replaced by a much more sophisticated resource provisioning system!
-	for resourceName, resourceSpec := range spec.Resources {
-		resClass := util.DerefOr(resourceSpec.Class, "default")
-		if resourceSpec.Type == "environment" {
-			if resClass != "default" {
-				return nil, nil, fmt.Errorf("resources.%s: '%s.%s' is not supported in score-compose", resourceName, resourceSpec.Type, resClass)
-			}
-			resources[resourceName] = envVarTracker
-		} else if resourceSpec.Type == "volume" && resClass == "default" {
-			resources[resourceName] = resourceWithStaticOutputs{}
-		} else {
-			slog.Warn(fmt.Sprintf("resources.%s: '%s.%s' is not directly supported in score-compose, references will be converted to environment variables", resourceName, resourceSpec.Type, resClass))
-			// TODO: only enable this if the type.class is in an allow-list or the allow-list is '*' - otherwise return an error
-			resources[resourceName] = envVarTracker.GenerateResource(resourceName)
-		}
-	}
-
 	ctx, err := buildContext(spec.Metadata, resources)
 	if err != nil {
-		return nil, nil, fmt.Errorf("preparing context: %w", err)
+		return nil, fmt.Errorf("preparing context: %w", err)
 	}
 
 	var ports []compose.ServicePortConfig
@@ -95,7 +75,7 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, *EnvVarTracker, error)
 		for key, val := range cSpec.Variables {
 			resolved, err := ctx.Substitute(val)
 			if err != nil {
-				return nil, nil, fmt.Errorf("containers.%s.variables.%s: %w", containerName, key, err)
+				return nil, fmt.Errorf("containers.%s.variables.%s: %w", containerName, key, err)
 			}
 			env[key] = &resolved
 		}
@@ -111,21 +91,21 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, *EnvVarTracker, error)
 			volumes = make([]compose.ServiceVolumeConfig, len(cSpec.Volumes))
 			for idx, vol := range cSpec.Volumes {
 				if vol.Path != nil && *vol.Path != "" {
-					return nil, nil, fmt.Errorf("containers.%s.volumes[%d].path: can't mount named volume with sub path '%s': not supported", containerName, idx, *vol.Path)
+					return nil, fmt.Errorf("containers.%s.volumes[%d].path: can't mount named volume with sub path '%s': not supported", containerName, idx, *vol.Path)
 				}
 
 				// TODO: deprecate this - volume should be linked directly
 				resolvedVolumeSource, err := ctx.Substitute(vol.Source)
 				if err != nil {
-					return nil, nil, fmt.Errorf("containers.%s.volumes[%d].source: %w", containerName, idx, err)
+					return nil, fmt.Errorf("containers.%s.volumes[%d].source: %w", containerName, idx, err)
 				} else if resolvedVolumeSource != vol.Source {
 					slog.Warn(fmt.Sprintf("containers.%s.volumes[%d].source: interpolation will be deprecated in the future", containerName, idx))
 				}
 
 				if res, ok := spec.Resources[resolvedVolumeSource]; !ok {
-					return nil, nil, fmt.Errorf("containers.%s.volumes[%d].source: resource '%s' does not exist", containerName, idx, resolvedVolumeSource)
+					return nil, fmt.Errorf("containers.%s.volumes[%d].source: resource '%s' does not exist", containerName, idx, resolvedVolumeSource)
 				} else if res.Type != "volume" {
-					return nil, nil, fmt.Errorf("containers.%s.volumes[%d].source: resource '%s' is not a volume", containerName, idx, resolvedVolumeSource)
+					return nil, fmt.Errorf("containers.%s.volumes[%d].source: resource '%s' is not a volume", containerName, idx, resolvedVolumeSource)
 				}
 
 				volumes[idx] = compose.ServiceVolumeConfig{
@@ -144,7 +124,7 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, *EnvVarTracker, error)
 
 		// Files are not supported just yet
 		if len(cSpec.Files) > 0 {
-			return nil, nil, fmt.Errorf("containers.%s.files: not supported", containerName)
+			return nil, fmt.Errorf("containers.%s.files: not supported", containerName)
 		}
 
 		// Docker compose without swarm/stack mode doesn't really support resource limits. There are optimistic
@@ -184,5 +164,5 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, *EnvVarTracker, error)
 		}
 		project.Services[svc.Name] = svc
 	}
-	return &project, envVarTracker, nil
+	return &project, nil
 }
diff --git a/internal/compose/templates.go b/internal/compose/templates.go
index a5388a6..3bf5fc7 100644
--- a/internal/compose/templates.go
+++ b/internal/compose/templates.go
@@ -14,6 +14,8 @@ import (
 	"maps"
 	"regexp"
 	"strings"
+
+	"github.com/score-spec/score-compose/internal/project"
 )
 
 var (
@@ -23,11 +25,11 @@ var (
 // templatesContext ia an utility type that provides a context for '${...}' templates substitution
 type templatesContext struct {
 	meta      resourceWithStaticOutputs
-	resources map[string]ResourceWithOutputs
+	resources map[string]project.OutputLookupFunc
 }
 
 // buildContext initializes a new templatesContext instance
-func buildContext(metadata map[string]interface{}, resources map[string]ResourceWithOutputs) (*templatesContext, error) {
+func buildContext(metadata map[string]interface{}, resources map[string]project.OutputLookupFunc) (*templatesContext, error) {
 	return &templatesContext{
 		meta:      maps.Clone(metadata),
 		resources: maps.Clone(resources),
@@ -93,7 +95,7 @@ func (ctx *templatesContext) mapVar(ref string) (string, error) {
 		} else if len(parts) == 2 {
 			// TODO: deprecate this - this is an annoying and nonsensical legacy thing
 			return parts[1], nil
-		} else if rv2, err := rv.LookupOutput(parts[2:]...); err != nil {
+		} else if rv2, err := rv(parts[2:]...); err != nil {
 			return "", fmt.Errorf("invalid ref '%s': %w", ref, err)
 		} else {
 			resolvedValue = rv2
diff --git a/internal/project/project.go b/internal/project/project.go
index 7e7488d..b4a3564 100644
--- a/internal/project/project.go
+++ b/internal/project/project.go
@@ -12,7 +12,8 @@ import (
 
 const (
 	DefaultRelativeStateDirectory = ".score-compose"
-	ConfigFileName                = "config.yaml"
+	StateFileName                 = "state.yaml"
+	MountsDirectoryName           = "mounts"
 )
 
 // The StateDirectory holds the local state of the score-compose project, including any configuration, extensions,
@@ -20,12 +21,8 @@ const (
 type StateDirectory struct {
 	// The path to the .score-compose directory
 	Path string
-	// The current config read from the config.yaml file
-	Config Config
-}
-
-type Config struct {
-	ComposeProjectName string `yaml:"composeProject"`
+	// The current state file
+	State State
 }
 
 // Persist ensures that the directory is created and that the current config file has been written with the latest settings.
@@ -36,18 +33,21 @@ func (sd *StateDirectory) Persist() error {
 	if err := os.Mkdir(sd.Path, 0755); err != nil && !errors.Is(err, os.ErrExist) {
 		return fmt.Errorf("failed to create directory '%s': %w", sd.Path, err)
 	}
+	if err := os.Mkdir(sd.State.MountsDirectory, 0755); err != nil && !errors.Is(err, os.ErrExist) {
+		return fmt.Errorf("failed to create mounts directory: %w", err)
+	}
 	out := new(bytes.Buffer)
 	enc := yaml.NewEncoder(out)
 	enc.SetIndent(2)
-	if err := enc.Encode(sd.Config); err != nil {
+	if err := enc.Encode(sd.State); err != nil {
 		return fmt.Errorf("failed to encode content: %w", err)
 	}
 
-	// important that we overwrite this file atomically via a inode move
-	if err := os.WriteFile(filepath.Join(sd.Path, ConfigFileName+".temp"), out.Bytes(), 0755); err != nil {
-		return fmt.Errorf("failed to write config: %w", err)
-	} else if err := os.Rename(filepath.Join(sd.Path, ConfigFileName+".temp"), filepath.Join(sd.Path, ConfigFileName)); err != nil {
-		return fmt.Errorf("failed to complete writing config: %w", err)
+	// important that we overwrite this file atomically via an inode move
+	if err := os.WriteFile(filepath.Join(sd.Path, StateFileName+".temp"), out.Bytes(), 0755); err != nil {
+		return fmt.Errorf("failed to write state: %w", err)
+	} else if err := os.Rename(filepath.Join(sd.Path, StateFileName+".temp"), filepath.Join(sd.Path, StateFileName)); err != nil {
+		return fmt.Errorf("failed to complete writing state: %w", err)
 	}
 	return nil
 }
@@ -55,19 +55,19 @@ func (sd *StateDirectory) Persist() error {
 // LoadStateDirectory loads the state directory for the given directory (usually PWD).
 func LoadStateDirectory(directory string) (*StateDirectory, bool, error) {
 	d := filepath.Join(directory, DefaultRelativeStateDirectory)
-	content, err := os.ReadFile(filepath.Join(d, ConfigFileName))
+	content, err := os.ReadFile(filepath.Join(d, StateFileName))
 	if err != nil {
 		if errors.Is(err, os.ErrNotExist) {
 			return nil, false, nil
 		}
-		return nil, true, fmt.Errorf("config file couldn't be read: %w", err)
+		return nil, true, fmt.Errorf("state file couldn't be read: %w", err)
 	}
 
-	var out Config
+	var out State
 	dec := yaml.NewDecoder(bytes.NewReader(content))
 	dec.KnownFields(true)
 	if err := dec.Decode(&out); err != nil {
-		return nil, true, fmt.Errorf("config file couldn't be decoded: %w", err)
+		return nil, true, fmt.Errorf("state file couldn't be decoded: %w", err)
 	}
 	return &StateDirectory{d, out}, true, nil
 }
diff --git a/internal/project/resource_uid.go b/internal/project/resource_uid.go
new file mode 100644
index 0000000..8b026f8
--- /dev/null
+++ b/internal/project/resource_uid.go
@@ -0,0 +1,32 @@
+package project
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/score-spec/score-compose/internal/util"
+)
+
+type ResourceUid string
+
+func NewResourceUid(workloadName string, resName string, resType string, resClass *string, resId *string) ResourceUid {
+	if resClass == nil {
+		resClass = util.Ref("default")
+	}
+	if resId != nil {
+		return ResourceUid(fmt.Sprintf("%s.%s#%s", resType, *resClass, *resId))
+	}
+	return ResourceUid(fmt.Sprintf("%s.%s#%s.%s", resType, *resClass, workloadName, resName))
+}
+
+func (r ResourceUid) Type() string {
+	return strings.SplitN(string(r), ".", 2)[0]
+}
+
+func (r ResourceUid) Class() string {
+	return strings.SplitN(strings.SplitN(string(r), "#", 2)[0], ".", 2)[1]
+}
+
+func (r ResourceUid) Id() string {
+	return strings.SplitN(string(r), "#", 2)[1]
+}
diff --git a/internal/project/state.go b/internal/project/state.go
new file mode 100644
index 0000000..0a0398d
--- /dev/null
+++ b/internal/project/state.go
@@ -0,0 +1,170 @@
+package project
+
+import (
+	"fmt"
+	"maps"
+	"reflect"
+
+	score "github.com/score-spec/score-go/types"
+)
+
+// State is the mega-structure that contains the state of our workload specifications and resources.
+// Score specs are added to this structure and it stores the current resource set.
+type State struct {
+	Workloads          map[string]ScoreWorkloadState      `yaml:"workloads"`
+	Resources          map[ResourceUid]ScoreResourceState `yaml:"resources"`
+	SharedState        map[string]interface{}             `yaml:"shared_state"`
+	ComposeProjectName string                             `yaml:"compose_project"`
+	MountsDirectory    string                             `yaml:"mounts_directory"`
+}
+
+type ScoreWorkloadState struct {
+	// Spec is the final score spec after all overrides and images have been set. This is a validated score file.
+	Spec score.Workload `yaml:"spec"`
+	// File is the source score file if known.
+	File *string `yaml:"file,omitempty"`
+}
+
+type ScoreResourceState struct {
+	// Type is the resource type.
+	Type string `yaml:"type"`
+	// Class is the resource class or 'default' if not provided.
+	Class string `yaml:"class"`
+	// Id is the generated id for the resource, either <workload>.<resName> or <shared>.<id>. This is tracked so that
+	// we can deduplicate and work out where a resource came from.
+	Id string `yaml:"id"`
+
+	Metadata map[string]interface{}
+	Params   map[string]interface{}
+
+	// ProvisionerUri is the resolved provisioner uri that should be found in the config. This is tracked so that
+	// we identify which provisioner was used for a particular instance of the resource.
+	ProvisionerUri string `yaml:"provisioner"`
+	// State is the internal state local to this resource. It will be persisted to disk when possible.
+	State map[string]interface{} `yaml:"state"`
+
+	// Outputs is the current set of outputs for the resource. This is the output of calling the provider. It doesn't
+	// get persisted to disk.
+	Outputs map[string]interface{} `yaml:"-"`
+	// OutputLookupFunc is function that allows certain in-process providers to defer any output generation. If this is
+	// not provided, it will fallback to using what's in the outputs.
+	OutputLookupFunc OutputLookupFunc `yaml:"-"`
+}
+
+type OutputLookupFunc func(keys ...string) (interface{}, error)
+
+// WithWorkload returns a new copy of State with the workload added, if the workload already exists with the same name
+// then it will be replaced.
+// This is not a deep copy, but any writes are executed in a copy-on-write manner to avoid modifying the source.
+func (s *State) WithWorkload(spec *score.Workload, filePath *string) (*State, error) {
+	out := *s
+	if s.Workloads == nil {
+		out.Workloads = make(map[string]ScoreWorkloadState)
+	} else {
+		out.Workloads = maps.Clone(s.Workloads)
+	}
+	out.Workloads[spec.Metadata["name"].(string)] = ScoreWorkloadState{
+		Spec: *spec,
+		File: filePath,
+	}
+	return &out, nil
+}
+
+// WithPrimedResources returns a new copy of State with all workload resources resolved to at least their initial type,
+// class and id. New resources will have an empty provider set. Existing resources will not be touched.
+// This is not a deep copy, but any writes are executed in a copy-on-write manner to avoid modifying the source.
+func (s *State) WithPrimedResources() (*State, error) {
+	out := *s
+	if s.Resources == nil {
+		out.Resources = make(map[ResourceUid]ScoreResourceState)
+	} else {
+		out.Resources = maps.Clone(s.Resources)
+	}
+
+	primedResourceUids := make(map[ResourceUid]bool)
+	for workloadName, workload := range s.Workloads {
+		for resName, res := range workload.Spec.Resources {
+			resUid := NewResourceUid(workloadName, resName, res.Type, res.Class, res.Id)
+			if existing, ok := out.Resources[resUid]; !ok {
+				out.Resources[resUid] = ScoreResourceState{
+					Type:     resUid.Type(),
+					Class:    resUid.Class(),
+					Id:       resUid.Id(),
+					Metadata: res.Metadata,
+					Params:   res.Params,
+					State:    map[string]interface{}{},
+				}
+				primedResourceUids[resUid] = true
+			} else if !primedResourceUids[resUid] {
+				existing.Metadata = res.Metadata
+				existing.Params = res.Params
+				out.Resources[resUid] = existing
+				primedResourceUids[resUid] = true
+			} else {
+				// multiple definitions of the same shared resource, let's check for conflicting params and metadata
+				if res.Params != nil {
+					if existing.Params != nil && !reflect.DeepEqual(existing.Params, res.Params) {
+						return nil, fmt.Errorf("resource '%s': multiple definitions with different params", resUid)
+					}
+					existing.Params = res.Params
+				}
+				if res.Metadata != nil {
+					if existing.Metadata != nil && !reflect.DeepEqual(existing.Metadata, res.Metadata) {
+						return nil, fmt.Errorf("resource '%s': multiple definitions with different metadata", resUid)
+					}
+					existing.Metadata = res.Metadata
+				}
+				out.Resources[resUid] = existing
+			}
+		}
+	}
+	return &out, nil
+}
+
+// GetResourceOutputForWorkload returns an output function per resource name in the given workload. This is for
+// passing into the compose translation context to resolve placeholder references.
+// This does not modify the state.
+func (s *State) GetResourceOutputForWorkload(workloadName string) (map[string]OutputLookupFunc, error) {
+	workload, ok := s.Workloads[workloadName]
+	if !ok {
+		return nil, fmt.Errorf("workload '%s': does not exist", workloadName)
+	}
+	out := make(map[string]OutputLookupFunc)
+
+	for resName, res := range workload.Spec.Resources {
+		resUid := NewResourceUid(workloadName, resName, res.Type, res.Class, res.Id)
+		state, ok := s.Resources[resUid]
+		if !ok {
+			return nil, fmt.Errorf("workload '%s': resource '%s' (%s) is not primed", workloadName, resName, resUid)
+		}
+		out[resName] = state.OutputLookup
+	}
+	return out, nil
+}
+
+// OutputLookup is a function which can traverse an outputs tree to find a resulting key, this defers to the embedded
+// output function if it exists.
+func (s *ScoreResourceState) OutputLookup(keys ...string) (interface{}, error) {
+	if s.OutputLookupFunc != nil {
+		return s.OutputLookupFunc(keys...)
+	} else if len(keys) == 0 {
+		return nil, fmt.Errorf("at least one lookup key is required")
+	}
+	var resolvedValue interface{}
+	resolvedValue = s.Outputs
+	for _, k := range keys {
+		ok := resolvedValue != nil
+		if ok {
+			var mapV map[string]interface{}
+			mapV, ok = resolvedValue.(map[string]interface{})
+			if !ok {
+				return "", fmt.Errorf("cannot lookup key '%s', context is not a map", k)
+			}
+			resolvedValue, ok = mapV[k]
+		}
+		if !ok {
+			return "", fmt.Errorf("key '%s' not found", k)
+		}
+	}
+	return resolvedValue, nil
+}


Test patch:
diff --git a/internal/command/init_test.go b/internal/command/init_test.go
index dc14df6..900f306 100644
--- a/internal/command/init_test.go
+++ b/internal/command/init_test.go
@@ -44,7 +44,11 @@ func TestInitNominal(t *testing.T) {
 	assert.NoError(t, err)
 	if assert.True(t, ok) {
 		assert.Equal(t, project.DefaultRelativeStateDirectory, sd.Path)
-		assert.Equal(t, filepath.Base(td), sd.Config.ComposeProjectName)
+		assert.Equal(t, filepath.Base(td), sd.State.ComposeProjectName)
+		assert.Equal(t, filepath.Join(project.DefaultRelativeStateDirectory, "mounts"), sd.State.MountsDirectory)
+		assert.Equal(t, map[string]project.ScoreWorkloadState{}, sd.State.Workloads)
+		assert.Equal(t, map[project.ResourceUid]project.ScoreResourceState{}, sd.State.Resources)
+		assert.Equal(t, map[string]interface{}{}, sd.State.SharedState)
 	}
 }
 
@@ -71,10 +75,25 @@ func TestInitNominal_custom_file_and_project(t *testing.T) {
 	assert.NoError(t, err)
 	if assert.True(t, ok) {
 		assert.Equal(t, project.DefaultRelativeStateDirectory, sd.Path)
-		assert.Equal(t, "bananas", sd.Config.ComposeProjectName)
+		assert.Equal(t, "bananas", sd.State.ComposeProjectName)
 	}
 }
 
+func TestInitNominal_bad_project(t *testing.T) {
+	td := t.TempDir()
+
+	wd, _ := os.Getwd()
+	require.NoError(t, os.Chdir(td))
+	defer func() {
+		require.NoError(t, os.Chdir(wd))
+	}()
+
+	stdout, stderr, err := executeAndResetCommand(context.Background(), rootCmd, []string{"init", "--project", "-this-is-invalid-"})
+	assert.EqualError(t, err, "invalid value for --project, it must match ^[a-z0-9][a-z0-9_-]*$")
+	assert.Equal(t, "", stdout)
+	assert.Equal(t, "", stderr)
+}
+
 func TestInitNominal_run_twice(t *testing.T) {
 	td := t.TempDir()
 
@@ -103,6 +122,10 @@ func TestInitNominal_run_twice(t *testing.T) {
 	assert.NoError(t, err)
 	if assert.True(t, ok) {
 		assert.Equal(t, project.DefaultRelativeStateDirectory, sd.Path)
-		assert.Equal(t, "bananas", sd.Config.ComposeProjectName)
+		assert.Equal(t, "bananas", sd.State.ComposeProjectName)
+		assert.Equal(t, filepath.Join(project.DefaultRelativeStateDirectory, "mounts"), sd.State.MountsDirectory)
+		assert.Equal(t, map[string]project.ScoreWorkloadState{}, sd.State.Workloads)
+		assert.Equal(t, map[project.ResourceUid]project.ScoreResourceState{}, sd.State.Resources)
+		assert.Equal(t, map[string]interface{}{}, sd.State.SharedState)
 	}
 }
diff --git a/internal/compose/convert_test.go b/internal/compose/convert_test.go
index 4dc19a9..2877bc5 100644
--- a/internal/compose/convert_test.go
+++ b/internal/compose/convert_test.go
@@ -15,6 +15,7 @@ import (
 	score "github.com/score-spec/score-go/types"
 	assert "github.com/stretchr/testify/assert"
 
+	"github.com/score-spec/score-compose/internal/project"
 	"github.com/score-spec/score-compose/internal/util"
 )
 
@@ -279,7 +280,13 @@ func TestScoreConvert(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.Name, func(t *testing.T) {
-			proj, vars, err := ConvertSpec(tt.Source)
+			evt := new(EnvVarTracker)
+			resourceOutputs := map[string]project.OutputLookupFunc{
+				"env":      evt.LookupOutput,
+				"app-db":   evt.GenerateResource("app-db").LookupOutput,
+				"some-dns": evt.GenerateResource("some-dns").LookupOutput,
+			}
+			proj, err := ConvertSpec(tt.Source, resourceOutputs)
 
 			if tt.Error != nil {
 				// On Error
@@ -290,7 +297,7 @@ func TestScoreConvert(t *testing.T) {
 				//
 				assert.NoError(t, err)
 				assert.Equal(t, tt.Project, proj)
-				assert.Equal(t, tt.Vars, vars.Accessed())
+				assert.Equal(t, tt.Vars, evt.Accessed())
 			}
 		})
 	}
diff --git a/internal/compose/templates_test.go b/internal/compose/templates_test.go
index 3ed67cb..3a37643 100644
--- a/internal/compose/templates_test.go
+++ b/internal/compose/templates_test.go
@@ -12,6 +12,8 @@ import (
 
 	score "github.com/score-spec/score-go/types"
 	assert "github.com/stretchr/testify/assert"
+
+	"github.com/score-spec/score-compose/internal/project"
 )
 
 func TestMapVar(t *testing.T) {
@@ -26,10 +28,10 @@ func TestMapVar(t *testing.T) {
 		}
 		return "", false
 	}
-	ctx, err := buildContext(meta, map[string]ResourceWithOutputs{
-		"env":    evt,
-		"db":     evt.GenerateResource("db"),
-		"static": resourceWithStaticOutputs{"x": "a"},
+	ctx, err := buildContext(meta, map[string]project.OutputLookupFunc{
+		"env":    evt.LookupOutput,
+		"db":     evt.GenerateResource("db").LookupOutput,
+		"static": resourceWithStaticOutputs{"x": "a"}.LookupOutput,
 	})
 	assert.NoError(t, err)
 
@@ -88,9 +90,9 @@ func TestSubstitute(t *testing.T) {
 		}
 		return "", false
 	}
-	ctx, err := buildContext(meta, map[string]ResourceWithOutputs{
-		"env": evt,
-		"db":  evt.GenerateResource("db"),
+	ctx, err := buildContext(meta, map[string]project.OutputLookupFunc{
+		"env": evt.LookupOutput,
+		"db":  evt.GenerateResource("db").LookupOutput,
 	})
 	assert.NoError(t, err)
 
diff --git a/internal/project/state_test.go b/internal/project/state_test.go
new file mode 100644
index 0000000..034cd9f
--- /dev/null
+++ b/internal/project/state_test.go
@@ -0,0 +1,208 @@
+package project
+
+import (
+	"testing"
+
+	score "github.com/score-spec/score-go/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func mustLoadWorkload(t *testing.T, spec string) *score.Workload {
+	t.Helper()
+	var raw score.Workload
+	require.NoError(t, yaml.Unmarshal([]byte(spec), &raw))
+	return &raw
+}
+
+func mustAddWorkload(t *testing.T, s *State, spec string) *State {
+	t.Helper()
+	w := mustLoadWorkload(t, spec)
+	n, err := s.WithWorkload(w, nil)
+	require.NoError(t, err)
+	return n
+}
+
+func TestWithWorkload(t *testing.T) {
+	start := new(State)
+
+	t.Run("one", func(t *testing.T) {
+		next, err := start.WithWorkload(mustLoadWorkload(t, `
+metadata:
+  name: example
+containers:
+  hello-world:
+    image: hi
+resources:
+  foo:
+    type: thing
+`), nil)
+		require.NoError(t, err)
+		assert.Len(t, start.Workloads, 0)
+		assert.Len(t, next.Workloads, 1)
+		assert.Nil(t, next.Workloads["example"].File, nil)
+		assert.Equal(t, score.Workload{
+			Metadata:   map[string]interface{}{"name": "example"},
+			Containers: map[string]score.Container{"hello-world": {Image: "hi"}},
+			Resources:  map[string]score.Resource{"foo": {Type: "thing"}},
+		}, next.Workloads["example"].Spec)
+	})
+
+	t.Run("two", func(t *testing.T) {
+		next1, err := start.WithWorkload(mustLoadWorkload(t, `
+metadata:
+  name: example1
+containers:
+  hello-world:
+    image: hi
+resources:
+  foo:
+    type: thing
+`), nil)
+		require.NoError(t, err)
+		next2, err := next1.WithWorkload(mustLoadWorkload(t, `
+metadata:
+  name: example2
+containers:
+  hello-world:
+    image: hi
+`), nil)
+		require.NoError(t, err)
+
+		assert.Len(t, start.Workloads, 0)
+		assert.Len(t, next1.Workloads, 1)
+		assert.Len(t, next2.Workloads, 2)
+	})
+}
+
+func TestWithPrimedResources(t *testing.T) {
+	start := new(State)
+
+	t.Run("empty", func(t *testing.T) {
+		next, err := start.WithPrimedResources()
+		require.NoError(t, err)
+		assert.Len(t, next.Resources, 0)
+	})
+
+	t.Run("one workload - nominal", func(t *testing.T) {
+		next := mustAddWorkload(t, start, `
+metadata: {"name": "example"}
+resources:
+  one:
+    type: thing
+  two:
+    type: thing2
+    class: banana
+  three:
+    type: thing3
+    class: apple
+    id: dog
+    metadata:
+      annotations:
+        foo: bar
+    params:
+      color: green
+  four:
+    type: thing4
+    id: elephant
+  five:
+    type: thing4
+    id: elephant
+    metadata:
+      x: y
+    params:
+      color: blue
+`)
+		next, err := next.WithPrimedResources()
+		require.NoError(t, err)
+		assert.Len(t, start.Resources, 0)
+		assert.Equal(t, map[ResourceUid]ScoreResourceState{
+			"thing.default#example.one": {Type: "thing", Class: "default", Id: "example.one", State: map[string]interface{}{}},
+			"thing2.banana#example.two": {Type: "thing2", Class: "banana", Id: "example.two", State: map[string]interface{}{}},
+			"thing3.apple#dog": {
+				Type: "thing3", Class: "apple", Id: "dog", State: map[string]interface{}{},
+				Metadata: map[string]interface{}{"annotations": score.ResourceMetadata{"foo": "bar"}},
+				Params:   map[string]interface{}{"color": "green"},
+			},
+			"thing4.default#elephant": {
+				Type: "thing4", Class: "default", Id: "elephant", State: map[string]interface{}{},
+				Metadata: map[string]interface{}{"x": "y"},
+				Params:   map[string]interface{}{"color": "blue"},
+			},
+		}, next.Resources)
+	})
+
+	t.Run("one workload - diff metadata", func(t *testing.T) {
+		next := mustAddWorkload(t, start, `
+metadata: {"name": "example"}
+resources:
+  one:
+    type: thing
+    id: elephant
+    metadata:
+      x: a
+  two:
+    type: thing
+    id: elephant
+    metadata:
+      x: y
+`)
+		next, err := next.WithPrimedResources()
+		require.EqualError(t, err, "resource 'thing.default#elephant': multiple definitions with different metadata")
+		assert.Len(t, start.Resources, 0)
+	})
+
+	t.Run("one workload - diff params", func(t *testing.T) {
+		next := mustAddWorkload(t, start, `
+metadata: {"name": "example"}
+resources:
+  one:
+    type: thing
+    id: elephant
+    params:
+      x: a
+  two:
+    type: thing
+    id: elephant
+    params:
+      x: y
+`)
+		next, err := next.WithPrimedResources()
+		require.EqualError(t, err, "resource 'thing.default#elephant': multiple definitions with different params")
+		assert.Len(t, start.Resources, 0)
+	})
+
+	t.Run("two workload - nominal", func(t *testing.T) {
+		t.Run("one workload - nominal", func(t *testing.T) {
+			next := mustAddWorkload(t, start, `
+metadata: {"name": "example1"}
+resources:
+  one:
+    type: thing
+  two:
+    type: thing2
+    id: dog
+`)
+			next = mustAddWorkload(t, next, `
+metadata: {"name": "example2"}
+resources:
+  one:
+    type: thing
+  two:
+    type: thing2
+    id: dog
+`)
+			next, err := next.WithPrimedResources()
+			require.NoError(t, err)
+			assert.Len(t, start.Resources, 0)
+			assert.Len(t, next.Resources, 3)
+			assert.Equal(t, map[ResourceUid]ScoreResourceState{
+				"thing.default#example1.one": {Type: "thing", Class: "default", Id: "example1.one", State: map[string]interface{}{}},
+				"thing.default#example2.one": {Type: "thing", Class: "default", Id: "example2.one", State: map[string]interface{}{}},
+				"thing2.default#dog":         {Type: "thing2", Class: "default", Id: "dog", State: map[string]interface{}{}},
+			}, next.Resources)
+		})
+	})
+
+}