Repository: bacalhau-project/bacalhau (language: python)

Issue:
Don't persist defaults in BACALHAU_DIR/config.yaml
When we initialize Bacalhau configuration for the very first time, we create a `config.yaml` file with default config values under `BACALHAU_DIR` ( usually `~/.bacalhau`). 

The problem is changing these defaults will only take affect with new users/hosts that run the CLI for the very first time. Existing users who did not explicitly override the configs will have to take a manual action to apply the new configs, which will result in undesired behaviours, such as https://github.com/bacalhau-project/bacalhau/issues/3003.

The right behaviour should be for `config.yaml` to be empty by default, and be a place for users to define their own configs. This should make `config.yaml` be less noisy, and enable updating defaults without user intervention 

The mitigation for now is for users to:
- If only using default configs, then delete `~/.bacalhau/config.yaml` 
- If overriding some configs, then delete entries inside `~/.bacalhau/config.yaml` that weren't provided by you


Discussion / hints:
ideally, we should bump the repo version from 1 to 2, and cleanup the config.yaml on behalf of the user as part of the migration. 

Fix patch:
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 61d8307461..49c7898ab2 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -2,15 +2,17 @@ package config
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/mitchellh/mapstructure"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v3"
 
-	"github.com/bacalhau-project/bacalhau/pkg/config/configenv"
+	"github.com/bacalhau-project/bacalhau/pkg/config/migrations"
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 )
 
@@ -18,6 +20,20 @@ const (
 	environmentVariablePrefix = "BACALHAU"
 	inferConfigTypes          = true
 	automaticEnvVar           = true
+
+	// user key files
+	Libp2pPrivateKeyFileName = "libp2p_private_key"
+	UserPrivateKeyFileName   = "user_id.pem"
+
+	// compute paths
+	ComputeStoragesPath = "executor_storages"
+	PluginsPath         = "plugins"
+
+	// requester paths
+	AutoCertCachePath = "autocert-cache"
+
+	// update check paths
+	UpdateCheckStatePath = "update.json"
 )
 
 var (
@@ -25,162 +41,144 @@ var (
 	configDecoderHook          = viper.DecodeHook(mapstructure.TextUnmarshallerHookFunc())
 )
 
-func WriteConfig(fileName string) error {
-	var cfg types.BacalhauConfig
-	if err := viper.Unmarshal(&cfg, configDecoderHook); err != nil {
-		return err
-	}
+const (
+	configType = "yaml"
+	configName = "config"
+)
 
-	cfgBytes, err := yaml.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-	flags := os.O_CREATE | os.O_TRUNC | os.O_WRONLY
-	f, err := os.OpenFile(fileName, flags, os.FileMode(0o644)) //nolint:gomnd
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	if _, err := f.Write(cfgBytes); err != nil {
-		return err
-	}
+func Init(path string) (types.BacalhauConfig, error) {
+	// derive the default config for the specified environment.
+	defaultConfig := ForEnvironment()
 
-	// read the config we wrote into viper, setting its values as the defaults used for configuration
-	return viper.ReadInConfig()
+	// set default values for path dependent config.
+	defaultConfig.User.KeyPath = filepath.Join(path, UserPrivateKeyFileName)
+	defaultConfig.User.Libp2pKeyPath = filepath.Join(path, Libp2pPrivateKeyFileName)
+	defaultConfig.Node.ExecutorPluginPath = filepath.Join(path, PluginsPath)
+	defaultConfig.Node.ComputeStoragePath = filepath.Join(path, ComputeStoragesPath)
+	defaultConfig.Update.CheckStatePath = filepath.Join(path, UpdateCheckStatePath)
+
+	// initialize the configuration with default values.
+	return initConfig(path, WithDefaultConfig(defaultConfig))
 }
 
-func ReadConfig(fileName string) error {
-	if _, err := os.Stat(fileName); os.IsNotExist(err) {
-		// if the config file doesn't exist that's fine, we will just use default configuration values
-		// dictated by the environment
+func Load(path string) (types.BacalhauConfig, error) {
+	// derive the default config for the specified environment.
+	defaultConfig := ForEnvironment()
+
+	// set default values for path dependent config.
+	defaultConfig.User.KeyPath = filepath.Join(path, UserPrivateKeyFileName)
+	defaultConfig.User.Libp2pKeyPath = filepath.Join(path, Libp2pPrivateKeyFileName)
+	defaultConfig.Node.ExecutorPluginPath = filepath.Join(path, PluginsPath)
+	defaultConfig.Node.ComputeStoragePath = filepath.Join(path, ComputeStoragesPath)
+	defaultConfig.Update.CheckStatePath = filepath.Join(path, UpdateCheckStatePath)
+
+	return initConfig(path, WithDefaultConfig(defaultConfig), WithFileHandler(ReadConfigHandler))
+}
+
+func Migrate(path string) error {
+	// check if the config file exists, if one is not found we don't need to migrate it
+	configPath := filepath.Join(path, "config.yaml")
+	if _, err := os.Stat(configPath); os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
-		return err
+		return fmt.Errorf("failed to stat config file for migration: %w", err)
 	}
-	// else we will read values set from the config, and accept those over the default values.
-	return viper.ReadInConfig()
-}
 
-func Init(defaultConfig types.BacalhauConfig, path string, fileName, fileType string) (types.BacalhauConfig, error) {
-	return initConfig(initParams{
-		filePath:      path,
-		fileName:      fileName,
-		fileType:      fileType,
-		fileHandler:   WriteConfig,
-		defaultConfig: defaultConfig,
-	})
-}
+	// open the config file
+	f, err := os.Open(configPath)
+	if err != nil {
+		return fmt.Errorf("failed to open config file at %q: %w", configPath, err)
+	}
 
-func Load(path string, fileName, fileType string) (types.BacalhauConfig, error) {
-	return initConfig(initParams{
-		filePath:      path,
-		fileName:      fileName,
-		fileType:      fileType,
-		fileHandler:   ReadConfig,
-		defaultConfig: ForEnvironment(),
-	})
-}
+	// read it all and unmarshal into yaml
+	b, err := io.ReadAll(f)
+	if err != nil {
+		if err := f.Close(); err != nil {
+			log.Error().Err(err).Msg("failed to close config file after failing to read it.")
+		}
+		return fmt.Errorf("failed to read config file: %w", err)
+	}
+	// we can close the file now that we read everything.
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to close config file: %w", err)
+	}
 
-func Get[T any](key string) (T, error) {
-	raw := viper.Get(key)
-	if raw == nil {
-		return zeroValue[T](), fmt.Errorf("value not found for %s", key)
+	var cfg types.BacalhauConfig
+	if err := yaml.Unmarshal(b, &cfg); err != nil {
+		return fmt.Errorf("failed to unmarshal config file: %w", err)
 	}
 
-	var val T
-	val, ok := raw.(T)
-	if !ok {
-		err := ForKey(key, &val)
+	// get all the migrations we need to apply to it.
+	migs, err := migrations.GetMigrations()
+	if err != nil {
+		return fmt.Errorf("failed to load migration list: %w", err)
+	}
+
+	// apply the migrations
+	currentCfg := cfg
+	for _, m := range migs {
+		log.Info().Msgf("applying migration sequence %d", m.Sequence())
+		currentCfg, err = m.Migrate(currentCfg)
 		if err != nil {
-			return zeroValue[T](), fmt.Errorf("value not of expected type, got: %T", raw)
+			return err
 		}
 	}
+	log.Info().Msgf("config migration complete")
 
-	return val, nil
-}
-
-func GetStringMapString(key string) map[string]string {
-	return viper.GetStringMapString(key)
-}
+	// marshal the migrated config back to yaml
+	marshaledCfg, err := yaml.Marshal(currentCfg)
+	if err != nil {
+		return fmt.Errorf("failed to marshal migrated config: %w", err)
+	}
 
-func zeroValue[T any]() T {
-	var zero T
-	return zero
-}
+	// open the file for writing and truncate it.
+	fw, err := os.OpenFile(configPath, os.O_WRONLY|os.O_TRUNC, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to open config file for writing at %q: %w", configPath, err)
+	}
+	defer fw.Close()
 
-// SetDefault sets the default value for the configuration.
-// Default only used when no value is provided by the user via an explicit call to Set, flag, config file or ENV.
-func SetDefault(config types.BacalhauConfig) error {
-	types.SetDefaults(config)
-	return nil
-}
+	// write the marshaled data back to the file
+	if _, err := fw.Write(marshaledCfg); err != nil {
+		return fmt.Errorf("failed to write migrated config to file: %w", err)
+	}
 
-// Set sets the configuration value.
-// Will be used instead of values obtained via flags, config file, ENV, default.
-// Useful for testing.
-func Set(config types.BacalhauConfig) error {
-	types.Set(config)
 	return nil
 }
 
-// Reset clears all configuration, useful for testing.
-func Reset() {
-	viper.Reset()
+type Params struct {
+	FileName      string
+	FileType      string
+	FileHandler   func(fileName string) error
+	DefaultConfig types.BacalhauConfig
 }
 
-// Getenv wraps os.Getenv and retrieves the value of the environment variable named by the config key.
-// It returns the value, which will be empty if the variable is not present.
-func Getenv(key string) string {
-	return os.Getenv(KeyAsEnvVar(key))
-}
-
-// KeyAsEnvVar returns the environment variable corresponding to a config key
-func KeyAsEnvVar(key string) string {
-	return strings.ToUpper(
-		fmt.Sprintf("%s_%s", environmentVariablePrefix, environmentVariableReplace.Replace(key)),
-	)
-}
-
-type initParams struct {
-	filePath      string
-	fileName      string
-	fileType      string
-	fileHandler   func(fileName string) error
-	defaultConfig types.BacalhauConfig
-}
+func initConfig(path string, opts ...Option) (types.BacalhauConfig, error) {
+	params := &Params{
+		FileName:      configName,
+		FileType:      configType,
+		FileHandler:   NoopConfigHandler,
+		DefaultConfig: ForEnvironment(),
+	}
 
-func ForEnvironment() types.BacalhauConfig {
-	env := GetConfigEnvironment()
-	switch env {
-	case EnvironmentProd:
-		return configenv.Production
-	case EnvironmentStaging:
-		return configenv.Staging
-	case EnvironmentDev:
-		return configenv.Development
-	case EnvironmentTest:
-		return configenv.Testing
-	case EnvironmentLocal:
-		return configenv.Local
-	default:
-		// this would indicate an error in the above logic of `GetEnvironment()`
-		return configenv.Local
+	for _, opt := range opts {
+		opt(params)
 	}
-}
 
-func initConfig(params initParams) (types.BacalhauConfig, error) {
-	viper.AddConfigPath(params.filePath)
-	viper.SetConfigName(params.fileName)
-	viper.SetConfigType(params.fileType)
+	viper.AddConfigPath(path)
+	viper.SetConfigName(params.FileName)
+	viper.SetConfigType(params.FileType)
 	viper.SetEnvPrefix(environmentVariablePrefix)
 	viper.SetTypeByDefaultValue(inferConfigTypes)
 	viper.SetEnvKeyReplacer(environmentVariableReplace)
-	if err := SetDefault(params.defaultConfig); err != nil {
+	if err := SetDefault(params.DefaultConfig); err != nil {
 		return types.BacalhauConfig{}, nil
 	}
-	if err := params.fileHandler(filepath.Join(params.filePath, fmt.Sprintf("%s.%s", params.fileName, params.fileType))); err != nil {
+
+	if err := params.FileHandler(filepath.Join(path, fmt.Sprintf("%s.%s", params.FileName, params.FileType))); err != nil {
 		return types.BacalhauConfig{}, err
 	}
+
 	if automaticEnvVar {
 		viper.AutomaticEnv()
 	}
@@ -193,116 +191,20 @@ func initConfig(params initParams) (types.BacalhauConfig, error) {
 	return out, nil
 }
 
-// ForKey unmarshals configuration values associated with a given key into the provided cfg structure.
-// It uses unmarshalCompositeKey internally to handle composite keys, ensuring values spread across
-// nested sub-keys are correctly populated into the cfg structure.
-//
-// Parameters:
-//   - key: The configuration key to retrieve values for.
-//   - cfg: The structure into which the configuration values will be unmarshaled.
-//
-// Returns:
-//   - An error if any occurred during unmarshaling; otherwise, nil.
-func ForKey(key string, cfg interface{}) error {
-	return unmarshalCompositeKey(key, cfg)
-}
-
-// unmarshalCompositeKey takes a key and an output structure to unmarshal into. It gets the
-// composite value associated with the given key and decodes it into the provided output structure.
-// It's especially useful when the desired value is not directly associated with the key, but
-// instead is spread across various nested sub-keys within the configuration.
-func unmarshalCompositeKey(key string, output interface{}) error {
-	compositeValue, isNested, err := getCompositeValue(key)
-	if err != nil {
-		return err
-	}
-	decoderConfig := &mapstructure.DecoderConfig{
-		DecodeHook: mapstructure.TextUnmarshallerHookFunc(),
-		Result:     output,
-		TagName:    "mapstructure", // This is the default struct tag name used by Viper.
-	}
-
-	decoder, err := mapstructure.NewDecoder(decoderConfig)
-	if err != nil {
-		return err
-	}
-
-	if isNested {
-		val, ok := compositeValue[key]
-		if !ok {
-			// NB(forrest): this case should never happen as we ensure all configuration values
-			// have a corresponding key via code gen. If this does occur it represents an error we need to debug.
-			return fmt.Errorf("CRITICAL ERROR: invalid configuration detected for key: %s. Config value not found", key)
-		}
-		return decoder.Decode(val)
-	}
-
-	return decoder.Decode(compositeValue)
+// Reset clears all configuration, useful for testing.
+func Reset() {
+	viper.Reset()
 }
 
-// getCompositeValue constructs a composite value for a given key. If the key directly corresponds
-// to a set value in Viper, it returns that, and false to indicate the value isn't nested under the key.
-// Otherwise, it collects all nested values under that key and returns them as a nested map and true
-// indicating the value is nested under the key.
-func getCompositeValue(key string) (map[string]interface{}, bool, error) {
-	var compositeValue map[string]interface{}
-
-	// Fetch directly if the exact key exists
-	if viper.IsSet(key) {
-		rawValue := viper.Get(key)
-		switch v := rawValue.(type) {
-		case map[string]interface{}:
-			compositeValue = v
-		default:
-			return map[string]interface{}{
-				key: rawValue,
-			}, true, nil
-		}
-	} else {
-		return nil, false, fmt.Errorf("configuration value not found for key: %s", key)
-	}
-
-	lowerKey := strings.ToLower(key)
-
-	// Prepare a map for faster key lookup.
-	viperKeys := viper.AllKeys()
-	keyMap := make(map[string]string, len(viperKeys))
-	for _, k := range viperKeys {
-		keyMap[strings.ToLower(k)] = k
-	}
-
-	// Build a composite map of values for keys nested under the provided key.
-	for lowerK, originalK := range keyMap {
-		if strings.HasPrefix(lowerK, lowerKey+".") {
-			parts := strings.Split(lowerK[len(lowerKey)+1:], ".")
-			if err := setNested(compositeValue, parts, viper.Get(originalK)); err != nil {
-				return nil, false, nil
-			}
-		}
-	}
-
-	return compositeValue, false, nil
+// Getenv wraps os.Getenv and retrieves the value of the environment variable named by the config key.
+// It returns the value, which will be empty if the variable is not present.
+func Getenv(key string) string {
+	return os.Getenv(KeyAsEnvVar(key))
 }
 
-// setNested is a recursive helper function that sets a value in a nested map based on a slice of keys.
-// It goes through each key, creating maps for each level as needed, and ultimately sets the value
-// in the innermost map.
-func setNested(m map[string]interface{}, keys []string, value interface{}) error {
-	if len(keys) == 1 {
-		m[keys[0]] = value
-		return nil
-	}
-
-	// If the next map level doesn't exist, create it.
-	if m[keys[0]] == nil {
-		m[keys[0]] = make(map[string]interface{})
-	}
-
-	// Cast the nested level to a map and return an error if the type assertion fails.
-	nestedMap, ok := m[keys[0]].(map[string]interface{})
-	if !ok {
-		return fmt.Errorf("key %s is not of type map[string]interface{}", keys[0])
-	}
-
-	return setNested(nestedMap, keys[1:], value)
+// KeyAsEnvVar returns the environment variable corresponding to a config key
+func KeyAsEnvVar(key string) string {
+	return strings.ToUpper(
+		fmt.Sprintf("%s_%s", environmentVariablePrefix, environmentVariableReplace.Replace(key)),
+	)
 }
diff --git a/pkg/config/environment.go b/pkg/config/environment.go
index ac22deb6c2..5a9520d632 100644
--- a/pkg/config/environment.go
+++ b/pkg/config/environment.go
@@ -6,6 +6,9 @@ import (
 
 	"github.com/rs/zerolog/log"
 	flag "github.com/spf13/pflag"
+
+	"github.com/bacalhau-project/bacalhau/pkg/config/configenv"
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 )
 
 type Environment string
@@ -31,6 +34,25 @@ func (e Environment) IsKnown() bool {
 	return false
 }
 
+func ForEnvironment() types.BacalhauConfig {
+	env := GetConfigEnvironment()
+	switch env {
+	case EnvironmentProd:
+		return configenv.Production
+	case EnvironmentStaging:
+		return configenv.Staging
+	case EnvironmentDev:
+		return configenv.Development
+	case EnvironmentTest:
+		return configenv.Testing
+	case EnvironmentLocal:
+		return configenv.Local
+	default:
+		// this would indicate an error in the above logic of `GetEnvironment()`
+		return configenv.Local
+	}
+}
+
 func GetConfigEnvironment() Environment {
 	env := Environment(os.Getenv("BACALHAU_ENVIRONMENT"))
 	if !env.IsKnown() {
diff --git a/pkg/config/getters.go b/pkg/config/getters.go
index f5ec5bbe37..6b5cfa31e7 100644
--- a/pkg/config/getters.go
+++ b/pkg/config/getters.go
@@ -1,9 +1,12 @@
 package config
 
 import (
+	"fmt"
 	"os"
+	"strings"
 	"time"
 
+	"github.com/mitchellh/mapstructure"
 	"github.com/multiformats/go-multiaddr"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
@@ -161,3 +164,146 @@ func GetStoragePath() string {
 func PreferredAddress() string {
 	return os.Getenv("BACALHAU_PREFERRED_ADDRESS")
 }
+
+func GetStringMapString(key string) map[string]string {
+	return viper.GetStringMapString(key)
+}
+
+func Get[T any](key string) (T, error) {
+	raw := viper.Get(key)
+	if raw == nil {
+		return zeroValue[T](), fmt.Errorf("value not found for %s", key)
+	}
+
+	var val T
+	val, ok := raw.(T)
+	if !ok {
+		err := ForKey(key, &val)
+		if err != nil {
+			return zeroValue[T](), fmt.Errorf("value not of expected type, got: %T: %w", raw, err)
+		}
+	}
+
+	return val, nil
+}
+
+func zeroValue[T any]() T {
+	var zero T
+	return zero
+}
+
+// ForKey unmarshals configuration values associated with a given key into the provided cfg structure.
+// It uses unmarshalCompositeKey internally to handle composite keys, ensuring values spread across
+// nested sub-keys are correctly populated into the cfg structure.
+//
+// Parameters:
+//   - key: The configuration key to retrieve values for.
+//   - cfg: The structure into which the configuration values will be unmarshaled.
+//
+// Returns:
+//   - An error if any occurred during unmarshaling; otherwise, nil.
+func ForKey(key string, cfg interface{}) error {
+	return unmarshalCompositeKey(key, cfg)
+}
+
+// unmarshalCompositeKey takes a key and an output structure to unmarshal into. It gets the
+// composite value associated with the given key and decodes it into the provided output structure.
+// It's especially useful when the desired value is not directly associated with the key, but
+// instead is spread across various nested sub-keys within the configuration.
+func unmarshalCompositeKey(key string, output interface{}) error {
+	compositeValue, isNested, err := getCompositeValue(key)
+	if err != nil {
+		return err
+	}
+	decoderConfig := &mapstructure.DecoderConfig{
+		DecodeHook: mapstructure.TextUnmarshallerHookFunc(),
+		Result:     output,
+		TagName:    "mapstructure", // This is the default struct tag name used by Viper.
+	}
+
+	decoder, err := mapstructure.NewDecoder(decoderConfig)
+	if err != nil {
+		return err
+	}
+
+	if isNested {
+		val, ok := compositeValue[key]
+		if !ok {
+			// NB(forrest): this case should never happen as we ensure all configuration values
+			// have a corresponding key via code gen. If this does occur it represents an error we need to debug.
+			err := fmt.Errorf("CRITICAL ERROR: invalid configuration detected for key: %s. Config value not found", key)
+			log.Err(err).Msg("invalid configuration detected")
+			return err
+		}
+		return decoder.Decode(val)
+	}
+
+	return decoder.Decode(compositeValue)
+}
+
+// getCompositeValue constructs a composite value for a given key. If the key directly corresponds
+// to a set value in Viper, it returns that, and false to indicate the value isn't nested under the key.
+// Otherwise, it collects all nested values under that key and returns them as a nested map and true
+// indicating the value is nested under the key.
+func getCompositeValue(key string) (map[string]interface{}, bool, error) {
+	var compositeValue map[string]interface{}
+
+	// Fetch directly if the exact key exists
+	if viper.IsSet(key) {
+		rawValue := viper.Get(key)
+		switch v := rawValue.(type) {
+		case map[string]interface{}:
+			compositeValue = v
+		default:
+			return map[string]interface{}{
+				key: rawValue,
+			}, true, nil
+		}
+	} else {
+		return nil, false, fmt.Errorf("configuration value not found for key: %s", key)
+	}
+
+	lowerKey := strings.ToLower(key)
+
+	// Prepare a map for faster key lookup.
+	viperKeys := viper.AllKeys()
+	keyMap := make(map[string]string, len(viperKeys))
+	for _, k := range viperKeys {
+		keyMap[strings.ToLower(k)] = k
+	}
+
+	// Build a composite map of values for keys nested under the provided key.
+	for lowerK, originalK := range keyMap {
+		if strings.HasPrefix(lowerK, lowerKey+".") {
+			parts := strings.Split(lowerK[len(lowerKey)+1:], ".")
+			if err := setNested(compositeValue, parts, viper.Get(originalK)); err != nil {
+				return nil, false, nil
+			}
+		}
+	}
+
+	return compositeValue, false, nil
+}
+
+// setNested is a recursive helper function that sets a value in a nested map based on a slice of keys.
+// It goes through each key, creating maps for each level as needed, and ultimately sets the value
+// in the innermost map.
+func setNested(m map[string]interface{}, keys []string, value interface{}) error {
+	if len(keys) == 1 {
+		m[keys[0]] = value
+		return nil
+	}
+
+	// If the next map level doesn't exist, create it.
+	if m[keys[0]] == nil {
+		m[keys[0]] = make(map[string]interface{})
+	}
+
+	// Cast the nested level to a map and return an error if the type assertion fails.
+	nestedMap, ok := m[keys[0]].(map[string]interface{})
+	if !ok {
+		return fmt.Errorf("key %s is not of type map[string]interface{}", keys[0])
+	}
+
+	return setNested(nestedMap, keys[1:], value)
+}
diff --git a/pkg/config/migrations/set.go b/pkg/config/migrations/set.go
new file mode 100644
index 0000000000..3a83a49daa
--- /dev/null
+++ b/pkg/config/migrations/set.go
@@ -0,0 +1,84 @@
+package migrations
+
+import (
+	"fmt"
+
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
+)
+
+var set migrationList
+
+func GetMigrations() ([]Migration, error) {
+	return set.GetMigrations()
+}
+
+func init() {
+	set = NewMigrationList()
+}
+
+type Migration interface {
+	Sequence() int
+	Migrate(config types.BacalhauConfig) (types.BacalhauConfig, error)
+}
+
+type mig struct {
+	seq       int
+	migration MigrationFn
+}
+
+func (m mig) Sequence() int {
+	return m.seq
+}
+
+func (m mig) Migrate(config types.BacalhauConfig) (types.BacalhauConfig, error) {
+	return m.migration(config)
+}
+
+type migrationList struct {
+	ms map[int]mig
+}
+
+func NewMigrationList() migrationList { // nolint: revive
+	return migrationList{map[int]mig{}}
+}
+
+// Register adds a migration to the migration list. This should be called in an init function.
+func (ml *migrationList) Register(seq int, mf MigrationFn) {
+	if seq <= 0 {
+		panic(fmt.Sprintf("invalid migration number: %d", seq))
+	}
+
+	if _, exists := ml.ms[seq]; exists {
+		panic(fmt.Sprintf("duplicate migration registered: %d", seq))
+	}
+
+	ml.ms[seq] = mig{
+		seq:       seq,
+		migration: mf,
+	}
+}
+
+type MigrationFn = func(config types.BacalhauConfig) (types.BacalhauConfig, error)
+
+func (ml *migrationList) GetMigrations() ([]Migration, error) {
+	// Check patch list is consistent with no gaps
+	count := len(ml.ms)
+
+	// migration 0 must not exist - it's the base schema by definition
+	if _, exists := ml.ms[0]; exists {
+		return nil, fmt.Errorf("found migration 0, which should not exist")
+	}
+
+	// index from 1 since schema seq 0 is the base and not in `ml`
+	for i := 1; i <= count; i++ {
+		if _, exists := ml.ms[i]; !exists {
+			return nil, fmt.Errorf("missing migration %d", i)
+		}
+	}
+
+	migs := make([]Migration, 0, count)
+	for i := 1; i <= count; i++ {
+		migs = append(migs, ml.ms[i])
+	}
+	return migs, nil
+}
diff --git a/pkg/config/migrations/v1_2.go b/pkg/config/migrations/v1_2.go
new file mode 100644
index 0000000000..c7f493e3d2
--- /dev/null
+++ b/pkg/config/migrations/v1_2.go
@@ -0,0 +1,61 @@
+package migrations
+
+import "github.com/bacalhau-project/bacalhau/pkg/config/types"
+
+var (
+	oldSwarmPeers = []string{
+		"/ip4/35.245.115.191/tcp/4001/p2p/12D3KooWE4wfAknWtY9mQ4eAA8zrFGeZa7X2Kh4nBP2tZgDSt7Rh",
+		"/ip4/35.245.115.191/udp/4001/quic/p2p/12D3KooWE4wfAknWtY9mQ4eAA8zrFGeZa7X2Kh4nBP2tZgDSt7Rh",
+		"/ip4/35.245.61.251/tcp/4001/p2p/12D3KooWD8zeukHTMyuPtQBoUUPqtEnaA7NwFXWcVywUJtCVPske",
+		"/ip4/35.245.61.251/udp/4001/quic/p2p/12D3KooWD8zeukHTMyuPtQBoUUPqtEnaA7NwFXWcVywUJtCVPske",
+		"/ip4/35.245.251.239/tcp/4001/p2p/12D3KooWAg1YdehZxcZhetcgA6KP8TLGX6Fq4h9PUswnUWoStVNc",
+		"/ip4/35.245.251.239/udp/4001/quic/p2p/12D3KooWAg1YdehZxcZhetcgA6KP8TLGX6Fq4h9PUswnUWoStVNc",
+		"/ip4/34.150.153.87/tcp/4001/p2p/12D3KooWGE4R98vokeLsRVdTv8D6jhMnifo81mm7NMRV8WJPNVHb",
+		"/ip4/34.150.153.87/udp/4001/quic/p2p/12D3KooWGE4R98vokeLsRVdTv8D6jhMnifo81mm7NMRV8WJPNVHb",
+		"/ip4/34.91.247.176/tcp/4001/p2p/12D3KooWSNKPM5PBchoqn774bpQ4j4QbL3VoyX6mH6vTyWXqE3kH",
+		"/ip4/34.91.247.176/udp/4001/quic/p2p/12D3KooWSNKPM5PBchoqn774bpQ4j4QbL3VoyX6mH6vTyWXqE3kH",
+	}
+
+	oldBootstrapPeers = []string{
+		"/ip4/35.245.115.191/tcp/1235/p2p/QmdZQ7ZbhnvWY1J12XYKGHApJ6aufKyLNSvf8jZBrBaAVL",
+		"/ip4/35.245.61.251/tcp/1235/p2p/QmXaXu9N5GNetatsvwnTfQqNtSeKAD6uCmarbh3LMRYAcF",
+		"/ip4/35.245.251.239/tcp/1235/p2p/QmYgxZiySj3MRkwLSL4X2MF5F9f2PMhAE3LV49XkfNL1o3",
+	}
+)
+
+func init() {
+	// this migration removes any IPFS swarm peers or Bootstrap peers that are incorrect from the v1.0.4 upgrade.
+	// if no incorrect values are present they are left as is.
+	set.Register(1, func(current types.BacalhauConfig) (types.BacalhauConfig, error) {
+		migrated := current
+		if haveSameElements(oldSwarmPeers, migrated.Node.IPFS.SwarmAddresses) {
+			migrated.Node.IPFS.SwarmAddresses = []string{}
+		}
+		if haveSameElements(oldBootstrapPeers, migrated.Node.BootstrapAddresses) {
+			migrated.Node.BootstrapAddresses = []string{}
+		}
+		return migrated, nil
+	})
+}
+
+// haveSameElements returns true if arr1 and arr2 have the same elements, false otherwise.
+func haveSameElements(arr1, arr2 []string) bool {
+	if len(arr1) != len(arr2) {
+		return false
+	}
+
+	elementCount := make(map[string]int)
+
+	for _, item := range arr1 {
+		elementCount[item]++
+	}
+
+	for _, item := range arr2 {
+		if count, exists := elementCount[item]; !exists || count == 0 {
+			return false
+		}
+		elementCount[item]--
+	}
+
+	return true
+}
diff --git a/pkg/config/options.go b/pkg/config/options.go
new file mode 100644
index 0000000000..c5cd26785d
--- /dev/null
+++ b/pkg/config/options.go
@@ -0,0 +1,76 @@
+package config
+
+import (
+	"os"
+
+	"github.com/spf13/viper"
+	"gopkg.in/yaml.v3"
+
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
+)
+
+type Option func(options *Params)
+
+func WithFileName(name string) Option {
+	return func(options *Params) {
+		options.FileName = name
+	}
+}
+
+func WithFileType(ftype string) Option {
+	return func(options *Params) {
+		options.FileType = ftype
+	}
+}
+
+func WithDefaultConfig(cfg types.BacalhauConfig) Option {
+	return func(options *Params) {
+		options.DefaultConfig = cfg
+	}
+}
+
+func WithFileHandler(handler func(name string) error) Option {
+	return func(options *Params) {
+		options.FileHandler = handler
+	}
+}
+
+func NoopConfigHandler(filename string) error {
+	return nil
+}
+
+func WriteConfigHandler(fileName string) error {
+	var cfg types.BacalhauConfig
+	if err := viper.Unmarshal(&cfg, configDecoderHook); err != nil {
+		return err
+	}
+
+	cfgBytes, err := yaml.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+	flags := os.O_CREATE | os.O_TRUNC | os.O_WRONLY
+	f, err := os.OpenFile(fileName, flags, os.FileMode(0o644)) //nolint:gomnd
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := f.Write(cfgBytes); err != nil {
+		return err
+	}
+
+	// read the config we wrote into viper, setting its values as the defaults used for configuration
+	return viper.ReadInConfig()
+}
+
+func ReadConfigHandler(fileName string) error {
+	if _, err := os.Stat(fileName); os.IsNotExist(err) {
+		// if the config file doesn't exist that's fine, we will just use default configuration values
+		// dictated by the environment
+		return nil
+	} else if err != nil {
+		return err
+	}
+	// else we will read values set from the config, and accept those over the default values.
+	return viper.ReadInConfig()
+}
diff --git a/pkg/config/setters.go b/pkg/config/setters.go
index a84c7eb7e0..f89436bdbd 100644
--- a/pkg/config/setters.go
+++ b/pkg/config/setters.go
@@ -6,30 +6,25 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 )
 
-func SetUserKey(path string) {
-	viper.Set(types.UserKeyPath, path)
-}
-
-func SetLibp2pKey(path string) {
-	viper.Set(types.UserLibp2pKeyPath, path)
-}
-
 func SetIntallationID(path string) {
 	viper.Set(types.UserInstallationID, path)
 }
 
-func SetExecutorPluginPath(path string) {
-	viper.Set(types.NodeExecutorPluginPath, path)
-}
-
-func SetComputeStoragesPath(path string) {
-	viper.Set(types.NodeComputeStoragePath, path)
+func SetUpdateCheckStatePath(path string) {
+	viper.Set(types.UpdateCheckStatePath, path)
 }
 
-func SetAutoCertCachePath(path string) {
-	viper.Set(types.NodeServerAPITLSAutoCertCachePath, path)
+// SetDefault sets the default value for the configuration.
+// Default only used when no value is provided by the user via an explicit call to Set, flag, config file or ENV.
+func SetDefault(config types.BacalhauConfig) error {
+	types.SetDefaults(config)
+	return nil
 }
 
-func SetUpdateCheckStatePath(path string) {
-	viper.Set(types.UpdateCheckStatePath, path)
+// Set sets the configuration value.
+// Will be used instead of values obtained via flags, config file, ENV, default.
+// Useful for testing.
+func Set(config types.BacalhauConfig) error {
+	types.Set(config)
+	return nil
 }
diff --git a/pkg/config/types/generated_constants.go b/pkg/config/types/generated_constants.go
index bc34c98099..a668ef56a9 100644
--- a/pkg/config/types/generated_constants.go
+++ b/pkg/config/types/generated_constants.go
@@ -97,6 +97,8 @@ const NodeRequesterWorkerWorkerCount = "Node.Requester.Worker.WorkerCount"
 const NodeRequesterWorkerWorkerEvalDequeueTimeout = "Node.Requester.Worker.WorkerEvalDequeueTimeout"
 const NodeRequesterWorkerWorkerEvalDequeueBaseBackoff = "Node.Requester.Worker.WorkerEvalDequeueBaseBackoff"
 const NodeRequesterWorkerWorkerEvalDequeueMaxBackoff = "Node.Requester.Worker.WorkerEvalDequeueMaxBackoff"
+const NodeRequesterStorageProvider = "Node.Requester.StorageProvider"
+const NodeRequesterStorageProviderS3 = "Node.Requester.StorageProvider.S3"
 const NodeRequesterStorageProviderS3PreSignedURLDisabled = "Node.Requester.StorageProvider.S3.PreSignedURLDisabled"
 const NodeRequesterStorageProviderS3PreSignedURLExpiration = "Node.Requester.StorageProvider.S3.PreSignedURLExpiration"
 const NodeBootstrapAddresses = "Node.BootstrapAddresses"
diff --git a/pkg/config/types/generated_viper_defaults.go b/pkg/config/types/generated_viper_defaults.go
index 04b3ab62cb..aea6880d05 100644
--- a/pkg/config/types/generated_viper_defaults.go
+++ b/pkg/config/types/generated_viper_defaults.go
@@ -100,6 +100,8 @@ func SetDefaults(cfg BacalhauConfig) {
 	viper.SetDefault(NodeRequesterWorkerWorkerEvalDequeueTimeout, cfg.Node.Requester.Worker.WorkerEvalDequeueTimeout.String())
 	viper.SetDefault(NodeRequesterWorkerWorkerEvalDequeueBaseBackoff, cfg.Node.Requester.Worker.WorkerEvalDequeueBaseBackoff.String())
 	viper.SetDefault(NodeRequesterWorkerWorkerEvalDequeueMaxBackoff, cfg.Node.Requester.Worker.WorkerEvalDequeueMaxBackoff.String())
+	viper.SetDefault(NodeRequesterStorageProvider, cfg.Node.Requester.StorageProvider)
+	viper.SetDefault(NodeRequesterStorageProviderS3, cfg.Node.Requester.StorageProvider.S3)
 	viper.SetDefault(NodeRequesterStorageProviderS3PreSignedURLDisabled, cfg.Node.Requester.StorageProvider.S3.PreSignedURLDisabled)
 	viper.SetDefault(NodeRequesterStorageProviderS3PreSignedURLExpiration, cfg.Node.Requester.StorageProvider.S3.PreSignedURLExpiration.String())
 	viper.SetDefault(NodeBootstrapAddresses, cfg.Node.BootstrapAddresses)
@@ -226,6 +228,8 @@ func Set(cfg BacalhauConfig) {
 	viper.Set(NodeRequesterWorkerWorkerEvalDequeueTimeout, cfg.Node.Requester.Worker.WorkerEvalDequeueTimeout.String())
 	viper.Set(NodeRequesterWorkerWorkerEvalDequeueBaseBackoff, cfg.Node.Requester.Worker.WorkerEvalDequeueBaseBackoff.String())
 	viper.Set(NodeRequesterWorkerWorkerEvalDequeueMaxBackoff, cfg.Node.Requester.Worker.WorkerEvalDequeueMaxBackoff.String())
+	viper.Set(NodeRequesterStorageProvider, cfg.Node.Requester.StorageProvider)
+	viper.Set(NodeRequesterStorageProviderS3, cfg.Node.Requester.StorageProvider.S3)
 	viper.Set(NodeRequesterStorageProviderS3PreSignedURLDisabled, cfg.Node.Requester.StorageProvider.S3.PreSignedURLDisabled)
 	viper.Set(NodeRequesterStorageProviderS3PreSignedURLExpiration, cfg.Node.Requester.StorageProvider.S3.PreSignedURLExpiration.String())
 	viper.Set(NodeBootstrapAddresses, cfg.Node.BootstrapAddresses)
diff --git a/pkg/repo/fs.go b/pkg/repo/fs.go
index ef4bf2354f..c1c0e5db83 100644
--- a/pkg/repo/fs.go
+++ b/pkg/repo/fs.go
@@ -2,48 +2,24 @@ package repo
 
 import (
 	"context"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
-	"encoding/base64"
-	"encoding/json"
-	"encoding/pem"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/google/uuid"
-	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/mitchellh/go-homedir"
 	"github.com/rs/zerolog/log"
 
 	"github.com/bacalhau-project/bacalhau/pkg/config"
-	"github.com/bacalhau-project/bacalhau/pkg/storage/util"
 	"github.com/bacalhau-project/bacalhau/pkg/telemetry"
 )
 
-type RepoVersion struct {
-	Version int
-}
-
 const (
-	// repo versioning
-	RepoVersion1    = 1
-	RepoVersionFile = "repo.version"
-
-	// user key files
-	Libp2pPrivateKeyFileName = "libp2p_private_key"
-	UserPrivateKeyFileName   = "user_id.pem"
-
-	// compute paths
-	ComputeStoragesPath = "executor_storages"
-	PluginsPath         = "plugins"
+	repoPermission         = 0755
+	defaultRunInfoFilename = "bacalhau.run"
+	runInfoFilePermissions = 0755
 
-	// requester paths
-	AutoCertCachePath = "autocert-cache"
-
-	// update check paths
+	// UpdateCheckStatePath is the update check paths.
 	UpdateCheckStatePath = "update.json"
 )
 
@@ -62,27 +38,6 @@ func NewFS(path string) (*FsRepo, error) {
 	}, nil
 }
 
-func (fsr *FsRepo) writeVersion() error {
-	repoVersion := RepoVersion{Version: RepoVersion1}
-	versionJSON, err := json.Marshal(repoVersion)
-	if err != nil {
-		return err
-	}
-	return os.WriteFile(filepath.Join(fsr.path, RepoVersionFile), versionJSON, util.OS_USER_RW)
-}
-
-func (fsr *FsRepo) readVersion() (int, error) {
-	versionBytes, err := os.ReadFile(filepath.Join(fsr.path, RepoVersionFile))
-	if err != nil {
-		return -1, err
-	}
-	var version RepoVersion
-	if err := json.Unmarshal(versionBytes, &version); err != nil {
-		return -1, err
-	}
-	return version.Version, nil
-}
-
 func (fsr *FsRepo) Path() (string, error) {
 	if exists, err := fsr.Exists(); err != nil {
 		return "", err
@@ -110,95 +65,13 @@ func (fsr *FsRepo) Exists() (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	if version != RepoVersion1 {
+	if version != RepoVersion1 && version != RepoVersion2 {
 		return false, fmt.Errorf("unknown repo version %d", version)
 	}
 	return true, nil
 }
 
-func (fsr *FsRepo) Open() error {
-	if exists, err := fsr.Exists(); err != nil {
-		return err
-	} else if !exists {
-		return fmt.Errorf("repo does not exist")
-	}
-
-	cfg, err := config.Load(fsr.path, configName, configType)
-	if err != nil {
-		return err
-	}
-
-	// TODO this should be a part of the config.
-	telemetry.SetupFromEnvs()
-
-	if cfg.User.KeyPath == "" {
-		// if the user has not specified the location of their user key via a config file, use the default value
-		cfg.User.KeyPath = filepath.Join(fsr.path, UserPrivateKeyFileName)
-		config.SetUserKey(cfg.User.KeyPath)
-	}
-
-	if cfg.User.Libp2pKeyPath == "" {
-		// if the user has not specified the location of their libp2p key via a config file, use the default value
-		cfg.User.Libp2pKeyPath = filepath.Join(fsr.path, Libp2pPrivateKeyFileName)
-		config.SetLibp2pKey(cfg.User.Libp2pKeyPath)
-	}
-
-	if cfg.User.InstallationID == "" {
-		ID, _ := config.GetClientID()
-		uuidFromUserID := uuid.NewSHA1(uuid.New(), []byte(ID))
-		config.SetIntallationID(uuidFromUserID.String())
-	}
-
-	if cfg.Node.ExecutorPluginPath == "" {
-		cfg.Node.ExecutorPluginPath = filepath.Join(fsr.path, PluginsPath)
-		config.SetExecutorPluginPath(cfg.Node.ExecutorPluginPath)
-	}
-
-	if cfg.Node.ServerAPI.TLS.AutoCertCachePath == "" {
-		cfg.Node.ServerAPI.TLS.AutoCertCachePath = filepath.Join(fsr.path, AutoCertCachePath)
-		config.SetAutoCertCachePath(cfg.Node.ServerAPI.TLS.AutoCertCachePath)
-	}
-
-	if cfg.Node.ComputeStoragePath == "" {
-		cfg.Node.ComputeStoragePath = filepath.Join(fsr.path, ComputeStoragesPath)
-		config.SetComputeStoragesPath(cfg.Node.ComputeStoragePath)
-	}
-
-	if cfg.Update.CheckStatePath == "" {
-		cfg.Update.CheckStatePath = filepath.Join(fsr.path, UpdateCheckStatePath)
-		config.SetUpdateCheckStatePath(cfg.Update.CheckStatePath)
-	}
-
-	// Using a slice of paths to minimize repetitive checks
-	pathsToCheck := []string{
-		cfg.User.KeyPath,
-		cfg.User.Libp2pKeyPath,
-		cfg.Node.ExecutorPluginPath,
-		cfg.Node.ComputeStoragePath,
-		// Note that cfg.Update.StateCheckPath is deliberately omitted here:
-		// this file will get created on first run of the update check
-	}
-
-	for _, path := range pathsToCheck {
-		if _, err := os.Stat(path); err != nil {
-			if os.IsNotExist(err) {
-				return fmt.Errorf("filepath '%s' does not exists", path)
-			}
-			return fmt.Errorf("failed to stat '%s': %w", path, err)
-		}
-	}
-
-	return nil
-}
-
-const (
-	configType     = "yaml"
-	configName     = "config"
-	repoPermission = 0755
-)
-
 func (fsr *FsRepo) Init() error {
-	defaultConfig := config.ForEnvironment()
 	if exists, err := fsr.Exists(); err != nil {
 		return err
 	} else if exists {
@@ -212,151 +85,69 @@ func (fsr *FsRepo) Init() error {
 		return err
 	}
 
-	var err error
-	defaultConfig.User.KeyPath, err = fsr.ensureUserIDKey(UserPrivateKeyFileName)
-	if err != nil {
-		return fmt.Errorf("failed to create user key: %w", err)
-	}
-	defaultConfig.User.Libp2pKeyPath, err = fsr.ensureLibp2pKey(Libp2pPrivateKeyFileName)
+	cfg, err := config.Init(fsr.path)
 	if err != nil {
-		return fmt.Errorf("failed to create libp2p key: %w", err)
-	}
-	defaultConfig.Node.ExecutorPluginPath, err = fsr.ensureDir(PluginsPath)
-	if err != nil {
-		return fmt.Errorf("failed to create plugin dir: %w", err)
-	}
-	defaultConfig.Node.ComputeStoragePath, err = fsr.ensureDir(ComputeStoragesPath)
-	if err != nil {
-		return fmt.Errorf("failed to create executor storage dir: %w", err)
+		return err
 	}
 
-	_, err = config.Init(defaultConfig, fsr.path, configName, configType)
-	if err != nil {
-		return err
+	if err := initRepoFiles(cfg); err != nil {
+		return fmt.Errorf("failed to initialize repo: %w", err)
 	}
 
 	// TODO this should be a part of the config.
 	telemetry.SetupFromEnvs()
-	return fsr.writeVersion()
+	return fsr.writeVersion(RepoVersion2)
 }
 
-func (fsr *FsRepo) ensureDir(name string) (string, error) {
-	path := filepath.Join(fsr.path, name)
-	if err := os.MkdirAll(path, util.OS_USER_RWX); err != nil {
-		return "", fmt.Errorf("failed to create directory at at '%s': %w", path, err)
+func (fsr *FsRepo) Open() error {
+	// if the repo does not exist we cannot open it.
+	if exists, err := fsr.Exists(); err != nil {
+		return err
+	} else if !exists {
+		return fmt.Errorf("repo does not exist")
 	}
-	return path, nil
-}
-
-const (
-	bitsPerKey = 2048 // number of bits in generated RSA keypairs
-)
 
-// ensureUserIDKey ensures that a default user ID key exists in the config dir.
-func (fsr *FsRepo) ensureUserIDKey(name string) (string, error) {
-	keyFile := filepath.Join(fsr.path, name)
-	if _, err := os.Stat(keyFile); err != nil {
-		if os.IsNotExist(err) {
-			log.Debug().Msgf(
-				"user ID key file '%s' does not exist, creating one", keyFile)
-
-			var key *rsa.PrivateKey
-			key, err = rsa.GenerateKey(rand.Reader, bitsPerKey)
-			if err != nil {
-				return "", fmt.Errorf("failed to generate private key: %w", err)
-			}
-
-			keyBytes := x509.MarshalPKCS1PrivateKey(key)
-			keyBlock := pem.Block{
-				Type:  "RSA PRIVATE KEY",
-				Bytes: keyBytes,
-			}
-
-			var file *os.File
-			file, err = os.Create(keyFile)
-			if err != nil {
-				return "", fmt.Errorf("failed to create key file: %w", err)
-			}
-			if err = pem.Encode(file, &keyBlock); err != nil {
-				return "", fmt.Errorf("failed to encode key file: %w", err)
-			}
-			if err = file.Close(); err != nil {
-				return "", fmt.Errorf("failed to close key file: %w", err)
-			}
-			if err = os.Chmod(keyFile, util.OS_USER_RW); err != nil {
-				return "", fmt.Errorf("failed to set permission on key file: %w", err)
-			}
-		} else {
-			return "", fmt.Errorf("failed to stat user ID key '%s': %w",
-				keyFile, err)
+	if repoVersion, err := fsr.readVersion(); err != nil {
+		return fmt.Errorf("failed to read repo version: %w", err)
+	} else if repoVersion == RepoVersion1 {
+		if err := config.Migrate(fsr.path); err != nil {
+			return fmt.Errorf("failed to migrate config: %w", err)
 		}
-	}
-
-	return keyFile, nil
-}
-
-func (fsr *FsRepo) ensureLibp2pKey(name string) (string, error) {
-	// We include the port in the filename so that in devstack multiple nodes
-	// running on the same host get different identities
-	privKeyPath := filepath.Join(fsr.path, name)
-
-	if _, err := os.Stat(privKeyPath); errors.Is(err, os.ErrNotExist) {
-		// Private key does not exist - create and write it
-
-		// Creates a new RSA key pair for this host.
-		prvKey, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, bitsPerKey, rand.Reader)
-		if err != nil {
-			log.Error().Err(err)
-			return "", err
-		}
-
-		keyOut, err := os.OpenFile(privKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, util.OS_USER_RW)
-		if err != nil {
-			return "", fmt.Errorf("failed to open key.pem for writing: %v", err)
-		}
-		privBytes, err := crypto.MarshalPrivateKey(prvKey)
-		if err != nil {
-			return "", fmt.Errorf("unable to marshal private key: %v", err)
+		if err := fsr.writeVersion(RepoVersion2); err != nil {
+			return fmt.Errorf("failed to update repo version: %w", err)
 		}
-		// base64 encode privBytes
-		b64 := base64.StdEncoding.EncodeToString(privBytes)
-		_, err = keyOut.WriteString(b64 + "\n")
-		if err != nil {
-			return "", fmt.Errorf("failed to write to key file: %v", err)
-		}
-		if err := keyOut.Close(); err != nil {
-			return "", fmt.Errorf("error closing key file: %v", err)
-		}
-		log.Debug().Msgf("wrote %s", privKeyPath)
-	} else {
-		return privKeyPath, err
 	}
 
-	// Now that we've ensured the private key is written to disk, read it! This
-	// ensures that loading it works even in the case where we've just created
-	// it.
-
-	// read the private key
-	keyBytes, err := os.ReadFile(privKeyPath)
+	// load the configuration for the repo.
+	cfg, err := config.Load(fsr.path)
 	if err != nil {
-		return "", fmt.Errorf("failed to read private key: %v", err)
+		return err
 	}
-	// base64 decode keyBytes
-	b64, err := base64.StdEncoding.DecodeString(string(keyBytes))
-	if err != nil {
-		return "", fmt.Errorf("failed to decode private key: %v", err)
+
+	// ensure the loaded config has valid fields as they pertain to the filesystem
+	// e.g. user key and libp2p files exists, storage paths exist, etc.
+	if err := validateRepoConfig(cfg); err != nil {
+		return fmt.Errorf("failed to validate repo config: %w", err)
 	}
-	// parse the private key
-	_, err = crypto.UnmarshalPrivateKey(b64)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse private key: %v", err)
+
+	// derive an installationID from the client ID loaded from the repo.
+	if cfg.User.InstallationID == "" {
+		ID, _ := config.GetClientID()
+		uuidFromUserID := uuid.NewSHA1(uuid.New(), []byte(ID))
+		config.SetIntallationID(uuidFromUserID.String())
 	}
 
-	return privKeyPath, nil
-}
+	// TODO we should be initializing the file as a part of creating the repo, instead of sometime later.
+	if cfg.Update.CheckStatePath == "" {
+		cfg.Update.CheckStatePath = filepath.Join(fsr.path, UpdateCheckStatePath)
+		config.SetUpdateCheckStatePath(cfg.Update.CheckStatePath)
+	}
 
-const defaultRunInfoFilename = "bacalhau.run"
-const runInfoFilePermissions = 0755
+	// TODO this should be a part of the config.
+	telemetry.SetupFromEnvs()
+
+	return nil
+}
 
 func (fsr *FsRepo) WriteRunInfo(ctx context.Context, summaryShellVariablesString string) (string, error) {
 	runInfoPath := filepath.Join(fsr.path, defaultRunInfoFilename)
diff --git a/pkg/repo/util.go b/pkg/repo/util.go
new file mode 100644
index 0000000000..2d28fac0b6
--- /dev/null
+++ b/pkg/repo/util.go
@@ -0,0 +1,218 @@
+package repo
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/pem"
+	"fmt"
+	"os"
+
+	"github.com/libp2p/go-libp2p/core/crypto"
+	"github.com/rs/zerolog/log"
+
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
+	"github.com/bacalhau-project/bacalhau/pkg/storage/util"
+)
+
+// initRepoFiles initializes all files required for a valid bacalhau repo.
+func initRepoFiles(cfg types.BacalhauConfig) error {
+	if err := initUserIDKey(cfg.User.KeyPath); err != nil {
+		return fmt.Errorf("failed to create user key: %w", err)
+	}
+
+	if err := initLibp2pKey(cfg.User.Libp2pKeyPath); err != nil {
+		return fmt.Errorf("failed to create libp2p key: %w", err)
+	}
+
+	if err := initDir(cfg.Node.ExecutorPluginPath); err != nil {
+		return fmt.Errorf("failed to create plugin dir: %w", err)
+	}
+
+	if err := initDir(cfg.Node.ComputeStoragePath); err != nil {
+		return fmt.Errorf("failed to create executor storage dir: %w", err)
+	}
+
+	return nil
+}
+
+// validateRepoConfig ensures all files exist for a valid bacalhau repo.
+func validateRepoConfig(cfg types.BacalhauConfig) error {
+	if exists, err := fileExists(cfg.User.KeyPath); err != nil {
+		return err
+	} else if !exists {
+		return fmt.Errorf("user key file does not exist at: %q", cfg.User.KeyPath)
+	}
+
+	if exists, err := fileExists(cfg.User.Libp2pKeyPath); err != nil {
+		return err
+	} else if !exists {
+		return fmt.Errorf("libp2p key file does not exist at: %q", cfg.User.Libp2pKeyPath)
+	}
+
+	if exists, err := fileExists(cfg.Node.ExecutorPluginPath); err != nil {
+		return err
+	} else if !exists {
+		return fmt.Errorf("executor plugin path does not exist at: %q", cfg.User.Libp2pKeyPath)
+	}
+
+	if exists, err := fileExists(cfg.Node.ComputeStoragePath); err != nil {
+		return err
+	} else if !exists {
+		return fmt.Errorf("compute storage path does not exist at: %q", cfg.Node.ComputeStoragePath)
+	}
+
+	return nil
+}
+
+func fileExists(path string) (bool, error) {
+	// Check if the file exists
+	_, err := os.Stat(path)
+	if err == nil {
+		// File exists
+		return true, nil
+	} else if !os.IsNotExist(err) {
+		// os.Stat returned an error other than "file does not exist"
+		return false, fmt.Errorf("failed to check if file exists at path: %w", err)
+	}
+	// file does not exist
+	return false, nil
+}
+
+// initDir will create a user directory at the specified path. It returns an error if a directory is already
+// present at the provided path.
+func initDir(path string) error {
+	// Check if the file exists, fail if it does, create it if it doesn't
+	exists, err := fileExists(path)
+	if err != nil {
+		return fmt.Errorf("failed to check dir at path: %w", err)
+	}
+	if exists {
+		// user key already exists.
+		return fmt.Errorf("dir already exists at path: %s", path)
+	}
+
+	if err := os.MkdirAll(path, util.OS_USER_RWX); err != nil {
+		return fmt.Errorf("failed to create directory at at '%s': %w", path, err)
+	}
+	return nil
+}
+
+const (
+	// bitsPerKey number of bits in generated RSA keypairs for the libp2p and user key.
+	bitsPerKey = 2048 // number of bits in generated RSA keypairs
+)
+
+// initUserIDKey will create a user key at the specified path. It returns an error if a file is already
+// present at the provided path.
+func initUserIDKey(path string) error {
+	// Check if the file exists, fail if it does, create it if it doesn't
+	exists, err := fileExists(path)
+	if err != nil {
+		return fmt.Errorf("failed to check user key file at path: %w", err)
+	}
+	if exists {
+		// user key already exists.
+		return fmt.Errorf("user key file already exists at path: %s", path)
+	}
+
+	// File does not exist, proceed with initialization
+	log.Debug().Msgf("initializing user ID key file at '%s'", path)
+
+	var key *rsa.PrivateKey
+	key, err = rsa.GenerateKey(rand.Reader, bitsPerKey)
+	if err != nil {
+		return fmt.Errorf("failed to generate private key: %w", err)
+	}
+
+	keyBytes := x509.MarshalPKCS1PrivateKey(key)
+	keyBlock := pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: keyBytes,
+	}
+
+	var file *os.File
+	file, err = os.Create(path)
+	if err != nil {
+		return fmt.Errorf("failed to create key file: %w", err)
+	}
+	defer file.Close()
+
+	if err = pem.Encode(file, &keyBlock); err != nil {
+		return fmt.Errorf("failed to encode key file: %w", err)
+	}
+
+	if err = os.Chmod(path, util.OS_USER_RW); err != nil {
+		return fmt.Errorf("failed to set permission on key file: %w", err)
+	}
+
+	return nil
+}
+
+// initLibp2pKey will create a libp2p key at the specified path. It returns an error if a file is already
+// present at the provided path.
+func initLibp2pKey(path string) error {
+	// Check if the file exists, fail if it does, create it if it doesn't
+	exists, err := fileExists(path)
+	if err != nil {
+		return fmt.Errorf("failed to check libp2p key file at path: %w", err)
+	}
+	if exists {
+		// user key already exists.
+		return fmt.Errorf("libp2p key file already exists at path: %s", path)
+	}
+
+	// File does not exist, proceed with initialization
+	log.Debug().Msgf("initializing libp2p key file at '%s'", path)
+
+	// Creates a new RSA key pair for this host.
+	prvKey, _, err := crypto.GenerateKeyPairWithReader(crypto.RSA, bitsPerKey, rand.Reader)
+	if err != nil {
+		log.Error().Err(err)
+		return err
+	}
+
+	keyOut, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, util.OS_USER_RW)
+	if err != nil {
+		return fmt.Errorf("failed to open key.pem for writing: %v", err)
+	}
+	privBytes, err := crypto.MarshalPrivateKey(prvKey)
+	if err != nil {
+		return fmt.Errorf("unable to marshal private key: %v", err)
+	}
+	// base64 encode privBytes
+	b64 := base64.StdEncoding.EncodeToString(privBytes)
+	_, err = keyOut.WriteString(b64 + "\n")
+	if err != nil {
+		return fmt.Errorf("failed to write to key file: %v", err)
+	}
+	if err := keyOut.Close(); err != nil {
+		return fmt.Errorf("error closing key file: %v", err)
+	}
+	log.Debug().Msgf("wrote %s", path)
+
+	// Now that we've ensured the private key is written to disk, read it! This
+	// ensures that loading it works even in the case where we've just created
+	// it.
+
+	{
+		// read the private key
+		keyBytes, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("failed to read private key: %v", err)
+		}
+		// base64 decode keyBytes
+		b64, err := base64.StdEncoding.DecodeString(string(keyBytes))
+		if err != nil {
+			return fmt.Errorf("failed to decode private key: %v", err)
+		}
+		// parse the private key
+		_, err = crypto.UnmarshalPrivateKey(b64)
+		if err != nil {
+			return fmt.Errorf("failed to parse private key: %v", err)
+		}
+	}
+
+	return nil
+}
diff --git a/pkg/repo/version.go b/pkg/repo/version.go
new file mode 100644
index 0000000000..d27b7d9f8a
--- /dev/null
+++ b/pkg/repo/version.go
@@ -0,0 +1,43 @@
+package repo
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/bacalhau-project/bacalhau/pkg/storage/util"
+)
+
+const (
+	// RepoVersion2 is the current repo versioning.
+	RepoVersion2 = 2
+	// RepoVersion1 is the current repo versioning for v1-v1.1.4
+	RepoVersion1 = 1
+	// RepoVersionFile is the name of the repo file containing the repo version.
+	RepoVersionFile = "repo.version"
+)
+
+type RepoVersion struct {
+	Version int
+}
+
+func (fsr *FsRepo) writeVersion(version int) error {
+	repoVersion := RepoVersion{Version: version}
+	versionJSON, err := json.Marshal(repoVersion)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(fsr.path, RepoVersionFile), versionJSON, util.OS_USER_RW)
+}
+
+func (fsr *FsRepo) readVersion() (int, error) {
+	versionBytes, err := os.ReadFile(filepath.Join(fsr.path, RepoVersionFile))
+	if err != nil {
+		return -1, err
+	}
+	var version RepoVersion
+	if err := json.Unmarshal(versionBytes, &version); err != nil {
+		return -1, err
+	}
+	return version.Version, nil
+}


Test patch:
diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
index a08cc6eb86..9ec4fa581a 100644
--- a/pkg/config/config_test.go
+++ b/pkg/config/config_test.go
@@ -3,6 +3,7 @@
 package config
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -69,20 +70,24 @@ func TestConfig(t *testing.T) {
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
 				defer Reset()
-				expectedConfig := configenv.Testing
+
+				// since BACALHAU_ENVIRONMENT is set to testing we expect the config to be test
+				require.NoError(t, os.Setenv("BACALHAU_ENVIRONMENT", "test"))
+				expected := configenv.Testing
+
 				configPath := t.TempDir()
 
-				_, err := Init(expectedConfig, configPath, tc.name, tc.configType)
+				_, err := Init(configPath)
 				require.NoError(t, err)
 
 				var out types.NodeConfig
 				err = ForKey(types.Node, &out)
 				assert.NoError(t, err)
-				assert.Equal(t, expectedConfig.Node.ServerAPI, out.ServerAPI)
+				assert.Equal(t, expected.Node.Requester, out.Requester)
 
-				retrieved, err := Get[string](types.NodeServerAPIHost)
-				assert.NoError(t, err)
-				assert.Equal(t, expectedConfig.Node.ServerAPI.Host, retrieved)
+				var retrieved types.RequesterConfig
+				require.NoError(t, ForKey(types.NodeRequester, &retrieved))
+				assert.Equal(t, expected.Node.Requester, retrieved)
 			})
 		}
 
@@ -100,29 +105,30 @@ func TestConfig(t *testing.T) {
 			t.Run(tc.name, func(t *testing.T) {
 				defer Reset()
 				// First, set up an expected configuration and save it using Init.
-				expectedConfig := configenv.Testing
+				require.NoError(t, os.Setenv("BACALHAU_ENVIRONMENT", "test"))
+				// since BACALHAU_ENVIRONMENT is set to testing we expect the config to be test
+				expected := configenv.Testing
 				configPath := t.TempDir()
-				configFile := "config"
 
-				_, err := Init(expectedConfig, configPath, configFile, tc.configType)
+				_, err := Init(configPath)
 				require.NoError(t, err)
 
 				// Now, try to load the configuration we just saved.
-				loadedConfig, err := Load(configPath, configFile, tc.configType)
+				loadedConfig, err := Load(configPath)
 				require.NoError(t, err)
 
 				// After loading, compare the loaded configuration with the expected configuration.
-				assert.Equal(t, expectedConfig.Node.ServerAPI, loadedConfig.Node.ServerAPI)
+				assert.Equal(t, expected.Node.Requester, loadedConfig.Node.Requester)
 
 				// Further, test specific parts:
-				var out types.APIConfig
-				err = ForKey(types.NodeServerAPI, &out)
+				var out types.NodeConfig
+				err = ForKey(types.Node, &out)
 				assert.NoError(t, err)
-				assert.Equal(t, expectedConfig.Node.ServerAPI, out)
+				assert.Equal(t, expected.Node.Requester, out.Requester)
 
-				retrieved, err := Get[string](types.NodeServerAPIHost)
-				assert.NoError(t, err)
-				assert.Equal(t, expectedConfig.Node.ServerAPI.Host, retrieved)
+				var retrieved types.RequesterConfig
+				require.NoError(t, ForKey(types.NodeRequester, &retrieved))
+				assert.Equal(t, expected.Node.Requester, retrieved)
 			})
 		}
 	})