Repository: kurtosis-tech/kurtosis
Pull request #1034 (2023-07-31 17:52:53)

feat: Persistent directories for docker
## Description:
Adds the docker backend to support persistent directories for user services.

## Is this change user facing?
NO
<!-- If yes, please add the "user facing" label to the PR -->
<!-- If yes, don't forget to include docs changes where relevant -->

## References (if applicable):
<!-- Add relevant Github Issues, Discord threads, or other helpful information. -->



Patch:
diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/files_artifacts_expansion.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/files_artifacts_expansion.go
index ec19b813d..d39ea26d4 100644
--- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/files_artifacts_expansion.go
+++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/files_artifacts_expansion.go
@@ -289,7 +289,7 @@ func createFilesArtifactsExpansionVolumes(
 		for key, value := range volumeAttrs.GetLabels() {
 			volumeLabelsStrs[key.GetString()] = value.GetString()
 		}
-		if err := dockerManager.CreateVolume(
+		if err = dockerManager.CreateVolume(
 			ctx,
 			volumeAttrs.GetName().GetString(),
 			volumeLabelsStrs,
@@ -302,23 +302,25 @@ func createFilesArtifactsExpansionVolumes(
 				mountpointExpanderWants,
 			)
 		}
-		//goland:noinspection GoDeferInLoop
-		defer func() {
-			if shouldDeleteVolumes {
-				// Background context so we still run this even if the input context was cancelled
-				if err := dockerManager.RemoveVolume(context.Background(), volumeNameStr); err != nil {
-					logrus.Warnf(
-						"Creating files artifact expansion volumes didn't complete successfully so we tried to delete volume '%v' that we created, but doing so threw an error:\n%v",
-						volumeNameStr,
-						err,
-					)
-					logrus.Warnf("You'll need to clean up volume '%v' manually!", volumeNameStr)
-				}
-			}
-		}()
-
 		result[mountpointExpanderWants] = volumeNameStr
 	}
+
+	defer func() {
+		if !shouldDeleteVolumes {
+			return
+		}
+		for _, volumeNameStr := range result {
+			// Background context so we still run this even if the input context was cancelled
+			if err := dockerManager.RemoveVolume(context.Background(), volumeNameStr); err != nil {
+				logrus.Warnf(
+					"Creating files artifact expansion volumes didn't complete successfully so we tried to delete volume '%v' that we created, but doing so threw an error:\n%v",
+					volumeNameStr,
+					err,
+				)
+				logrus.Warnf("You'll need to clean up volume '%v' manually!", volumeNameStr)
+			}
+		}
+	}()
 	shouldDeleteVolumes = false
 	return result, nil
 }
diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/persistent_directories.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/persistent_directories.go
new file mode 100644
index 000000000..a0dbbfbb6
--- /dev/null
+++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/persistent_directories.go
@@ -0,0 +1,78 @@
+package user_service_functions
+
+import (
+	"context"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
+	"github.com/kurtosis-tech/stacktrace"
+	"github.com/sirupsen/logrus"
+)
+
+func getOrCreatePersistentDirectories(
+	ctx context.Context,
+	serviceUuid service.ServiceUUID,
+	objAttrsProvider object_attributes_provider.DockerEnclaveObjectAttributesProvider,
+	serviceMountpointsToPersistentKey map[string]service_directory.DirectoryPersistentKey,
+	dockerManager *docker_manager.DockerManager,
+) (map[string]string, error) {
+	shouldDeleteVolumes := true
+	volumeNamesToRemoveIfFailure := map[string]bool{}
+	persistentDirectories := map[string]string{}
+
+	for serviceDirPath, persistentKey := range serviceMountpointsToPersistentKey {
+		volumeAttrs, err := objAttrsProvider.ForSinglePersistentDirectoryVolume(serviceUuid, persistentKey)
+		if err != nil {
+			return nil, stacktrace.Propagate(err, "Error creating persistent directory labels for '%s'", persistentKey)
+		}
+
+		volumeName := volumeAttrs.GetName().GetString()
+		volumeLabelsStrs := map[string]string{}
+		for key, value := range volumeAttrs.GetLabels() {
+			volumeLabelsStrs[key.GetString()] = value.GetString()
+		}
+
+		potentiallyExistingVolumes, err := dockerManager.GetVolumesByName(ctx, volumeName)
+		if err != nil {
+			return nil, stacktrace.Propagate(err, "An error occurred checking for persistent volume existence")
+		}
+		if len(potentiallyExistingVolumes) == 1 {
+			// volume already existed for this enclave, return it and continue to the next one
+			persistentDirectories[volumeName] = serviceDirPath
+			continue
+		} else if len(potentiallyExistingVolumes) > 1 {
+			return nil, stacktrace.NewError("More than one volume with name '%s' exists in docker. This is unexpected", volumeName)
+		}
+
+		if err = dockerManager.CreateVolume(ctx, volumeName, volumeLabelsStrs); err != nil {
+			return nil, stacktrace.Propagate(
+				err,
+				"An error occurred creating persistent directory volume '%s' for service '%v'",
+				persistentKey,
+				serviceUuid,
+			)
+		}
+		volumeNamesToRemoveIfFailure[volumeName] = true
+		persistentDirectories[volumeName] = serviceDirPath
+	}
+
+	defer func() {
+		if !shouldDeleteVolumes {
+			return
+		}
+		for volumeNameStr := range volumeNamesToRemoveIfFailure {
+			// Background context so we still run this even if the input context was cancelled
+			if err := dockerManager.RemoveVolume(context.Background(), volumeNameStr); err != nil {
+				logrus.Warnf(
+					"Creating persistent directory volumes didn't complete successfully so we tried to delete volume '%v' that we created, but doing so threw an error:\n%v",
+					volumeNameStr,
+					err,
+				)
+				logrus.Warnf("You'll need to clean up volume '%v' manually!", volumeNameStr)
+			}
+		}
+	}()
+	shouldDeleteVolumes = false
+	return persistentDirectories, nil
+}
diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go
index 9c64617c2..edf6d4a7e 100644
--- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go
+++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go
@@ -439,6 +439,7 @@ func createStartServiceOperation(
 
 	return func() (interface{}, error) {
 		filesArtifactsExpansion := serviceConfig.GetFilesArtifactsExpansion()
+		persistentDirectories := serviceConfig.GetPersistentDirectories()
 		containerImageName := serviceConfig.GetContainerImageName()
 		privatePorts := serviceConfig.GetPrivatePorts()
 		publicPorts := serviceConfig.GetPublicPorts()
@@ -481,7 +482,7 @@ func createStartServiceOperation(
 			defer func() {
 				if shouldDeleteVolumes {
 					for volumeName := range candidateVolumeMounts {
-						// Use background context so we delete these even if input context was cancelled
+						// Use background context, so we delete these even if input context was cancelled
 						if err := dockerManager.RemoveVolume(context.Background(), volumeName); err != nil {
 							logrus.Errorf("Starting the service failed so we tried to delete files artifact expansion volume '%v' that we created, but doing so threw an error:\n%v", volumeName, err)
 							logrus.Errorf("You'll need to delete volume '%v' manually!", volumeName)
@@ -489,7 +490,43 @@ func createStartServiceOperation(
 					}
 				}
 			}()
-			volumeMounts = candidateVolumeMounts
+
+			for dirpath, volumeName := range candidateVolumeMounts {
+				if _, found := volumeMounts[dirpath]; found {
+					return nil, stacktrace.NewError("An error occurred doing files artifacts expansion. Multiple volumes were mounted on the same path.")
+				}
+				volumeMounts[dirpath] = volumeName
+			}
+		}
+
+		if persistentDirectories != nil {
+			candidateVolumeMounts, err := getOrCreatePersistentDirectories(
+				ctx,
+				serviceUUID,
+				enclaveObjAttrsProvider,
+				persistentDirectories.ServiceDirpathToDirectoryPersistentKey,
+				dockerManager,
+			)
+			if err != nil {
+				return nil, stacktrace.Propagate(err, "An error occurred creating persistent directory volumes")
+			}
+			defer func() {
+				if shouldDeleteVolumes {
+					for volumeName := range candidateVolumeMounts {
+						// Use background context, so we delete these even if input context was cancelled
+						if err := dockerManager.RemoveVolume(context.Background(), volumeName); err != nil {
+							logrus.Errorf("Starting the service failed so we tried to delete persistent directory volume '%v' that we created, but doing so threw an error:\n%v", volumeName, err)
+							logrus.Errorf("You'll need to delete volume '%v' manually!", volumeName)
+						}
+					}
+				}
+			}()
+			for dirpath, volumeName := range candidateVolumeMounts {
+				if _, found := volumeMounts[dirpath]; found {
+					return nil, stacktrace.NewError("An error occurred creating persistent directory volumes. Multiple volumes were mounted on the same path.")
+				}
+				volumeMounts[dirpath] = volumeName
+			}
 		}
 
 		containerAttrs, err := enclaveObjAttrsProvider.ForUserServiceContainer(
@@ -548,6 +585,8 @@ func createStartServiceOperation(
 			skipAddingUserServiceToBridgeNetwork,
 		).WithContainerInitEnabled(
 			true,
+		).WithVolumeMounts(
+			volumeMounts,
 		)
 
 		if entrypointArgs != nil {
@@ -556,9 +595,6 @@ func createStartServiceOperation(
 		if cmdArgs != nil {
 			createAndStartArgsBuilder.WithCmdArgs(cmdArgs)
 		}
-		if volumeMounts != nil {
-			createAndStartArgsBuilder.WithVolumeMounts(volumeMounts)
-		}
 
 		createAndStartArgs := createAndStartArgsBuilder.Build()
 
diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go
index 246870085..e8631c7e0 100644
--- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go
+++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go
@@ -1,6 +1,8 @@
 package object_attributes_provider
 
 import (
+	"crypto/md5"
+	"encoding/hex"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_value"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_object_name"
@@ -9,6 +11,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator"
 	"github.com/kurtosis-tech/stacktrace"
 	"net"
@@ -19,7 +22,10 @@ import (
 const (
 	apiContainerNamePrefix                 = "kurtosis-api"
 	networkingSidecarContainerNameFragment = "networking-sidecar"
+
 	artifactExpansionVolumeNameFragment    = "files-artifact-expansion"
+	persistentServiceDirectoryNameFragment = "service-persistent-directory"
+
 	artifactsExpanderContainerNameFragment = "files-artifacts-expander"
 	logsCollectorFragment                  = "kurtosis-logs-collector"
 	// The collector is per enclave so this is a suffix
@@ -49,6 +55,10 @@ type DockerEnclaveObjectAttributesProvider interface {
 	ForSingleFilesArtifactExpansionVolume(
 		serviceUUID service.ServiceUUID,
 	) (DockerObjectAttributes, error)
+	ForSinglePersistentDirectoryVolume(
+		serviceUUID service.ServiceUUID,
+		persistentKey service_directory.DirectoryPersistentKey,
+	) (DockerObjectAttributes, error)
 	ForLogsCollector(tcpPortId string, tcpPortSpec *port_spec.PortSpec, httpPortId string, httpPortSpec *port_spec.PortSpec) (DockerObjectAttributes, error)
 	ForLogsCollectorVolume() (DockerObjectAttributes, error)
 }
@@ -321,6 +331,55 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForSingleFilesArtifac
 	return objectAttributes, nil
 }
 
+// In Docker we get one volume per persistent directory
+func (provider *dockerEnclaveObjectAttributesProviderImpl) ForSinglePersistentDirectoryVolume(
+	serviceUUID service.ServiceUUID,
+	persistentKey service_directory.DirectoryPersistentKey,
+) (
+	DockerObjectAttributes,
+	error,
+) {
+	serviceUuidStr := string(serviceUUID)
+
+	guidStr, err := uuid_generator.GenerateUUIDString()
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred generating a UUID for the persistent directory volume for service '%v'", serviceUuidStr)
+	}
+
+	hasher := md5.New()
+	hasher.Write([]byte(serviceUUID))
+	hasher.Write([]byte(persistentKey))
+	persistentKeyHash := hex.EncodeToString(hasher.Sum(nil))
+
+	name, err := provider.getNameForEnclaveObject([]string{
+		persistentServiceDirectoryNameFragment,
+		persistentKeyHash,
+	})
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred creating the files artifact expansion volume name object using GUID '%v' and service GUID '%v'", guidStr, serviceUuidStr)
+	}
+
+	labels, err := provider.getLabelsForEnclaveObjectWithGUID(guidStr)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred getting labels for files artifact expansion volume with UUID '%v'", guidStr)
+	}
+
+	serviceUuidLabelValue, err := docker_label_value.CreateNewDockerLabelValue(serviceUuidStr)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred creating a Docker label value from service GUID string '%v'", serviceUuidStr)
+	}
+	labels[label_key_consts.UserServiceGUIDDockerLabelKey] = serviceUuidLabelValue
+	labels[label_key_consts.VolumeTypeDockerLabelKey] = label_value_consts.PersistentDirectoryVolumeTypeDockerLabelValue
+	// TODO Create a KurtosisResourceDockerLabelKey object, like Kubernetes, and apply the "user-service" label here?
+
+	objectAttributes, err := newDockerObjectAttributesImpl(name, labels)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred while creating the ObjectAttributesImpl with the name '%s' and labels '%+v'", name, labels)
+	}
+
+	return objectAttributes, nil
+}
+
 // We'll have at most one files artifact expansion container per service, because the single container will handle
 // all expansion
 func (provider *dockerEnclaveObjectAttributesProviderImpl) ForFilesArtifactsExpanderContainer(
diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go
index 28d11a93a..9ace3728a 100644
--- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go
+++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go
@@ -24,6 +24,7 @@ const (
 
 	enclaveDataVolumeTypeLabelValueStr            = "enclave-data"
 	filesArtifactExpansionVolumeTypeLabelValueStr = "files-artifacts-expansion"
+	persistentDirectoryVolumeTypeLabelValueStr    = "persistent-directory"
 	logsDatabaseVolumeTypeLabelValueStr           = "logs-db"
 	logsCollectorVolumeTypeLabelValueStr          = "logs-collector-data"
 
@@ -53,5 +54,6 @@ var FilesArtifactExpanderContainerTypeDockerLabelValue = docker_label_value.Must
 
 var EnclaveDataVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(enclaveDataVolumeTypeLabelValueStr)
 var FilesArtifactExpansionVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(filesArtifactExpansionVolumeTypeLabelValueStr)
+var PersistentDirectoryVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(persistentDirectoryVolumeTypeLabelValueStr)
 var LogsDatabaseVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsDatabaseVolumeTypeLabelValueStr)
 var LogsCollectorVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsCollectorVolumeTypeLabelValueStr)
diff --git a/container-engine-lib/lib/backend_interface/objects/service/service_config.go b/container-engine-lib/lib/backend_interface/objects/service/service_config.go
index b8c33f628..762afd644 100644
--- a/container-engine-lib/lib/backend_interface/objects/service/service_config.go
+++ b/container-engine-lib/lib/backend_interface/objects/service/service_config.go
@@ -1,8 +1,8 @@
 package service
 
 import (
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/files_artifacts_expansion"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 )
 
 // Config options for the underlying container of a service
@@ -20,7 +20,9 @@ type ServiceConfig struct {
 	envVars map[string]string
 
 	// Leave as nil to not do any files artifact expansion
-	filesArtifactExpansion *files_artifacts_expansion.FilesArtifactsExpansion
+	filesArtifactExpansion *service_directory.FilesArtifactsExpansion
+
+	persistentDirectories *service_directory.PersistentDirectories
 
 	cpuAllocationMillicpus uint64
 
@@ -42,7 +44,8 @@ func NewServiceConfig(
 	entrypointArgs []string,
 	cmdArgs []string,
 	envVars map[string]string,
-	filesArtifactExpansion *files_artifacts_expansion.FilesArtifactsExpansion,
+	filesArtifactExpansion *service_directory.FilesArtifactsExpansion,
+	persistentDirectories *service_directory.PersistentDirectories,
 	cpuAllocationMillicpus uint64,
 	memoryAllocationMegabytes uint64,
 	privateIPAddrPlaceholder string,
@@ -58,6 +61,7 @@ func NewServiceConfig(
 		cmdArgs:                   cmdArgs,
 		envVars:                   envVars,
 		filesArtifactExpansion:    filesArtifactExpansion,
+		persistentDirectories:     persistentDirectories,
 		cpuAllocationMillicpus:    cpuAllocationMillicpus,
 		memoryAllocationMegabytes: memoryAllocationMegabytes,
 		privateIPAddrPlaceholder:  privateIPAddrPlaceholder,
@@ -92,10 +96,14 @@ func (serviceConfig *ServiceConfig) GetEnvVars() map[string]string {
 	return serviceConfig.envVars
 }
 
-func (serviceConfig *ServiceConfig) GetFilesArtifactsExpansion() *files_artifacts_expansion.FilesArtifactsExpansion {
+func (serviceConfig *ServiceConfig) GetFilesArtifactsExpansion() *service_directory.FilesArtifactsExpansion {
 	return serviceConfig.filesArtifactExpansion
 }
 
+func (serviceConfig *ServiceConfig) GetPersistentDirectories() *service_directory.PersistentDirectories {
+	return serviceConfig.persistentDirectories
+}
+
 func (serviceConfig *ServiceConfig) GetCPUAllocationMillicpus() uint64 {
 	return serviceConfig.cpuAllocationMillicpus
 }
diff --git a/container-engine-lib/lib/backend_interface/objects/files_artifacts_expansion/files_artifacts_expansion.go b/container-engine-lib/lib/backend_interface/objects/service_directory/files_artifacts_expansion.go
similarity index 95%
rename from container-engine-lib/lib/backend_interface/objects/files_artifacts_expansion/files_artifacts_expansion.go
rename to container-engine-lib/lib/backend_interface/objects/service_directory/files_artifacts_expansion.go
index 948042201..7d92e85fb 100644
--- a/container-engine-lib/lib/backend_interface/objects/files_artifacts_expansion/files_artifacts_expansion.go
+++ b/container-engine-lib/lib/backend_interface/objects/service_directory/files_artifacts_expansion.go
@@ -1,4 +1,4 @@
-package files_artifacts_expansion
+package service_directory
 
 type FilesArtifactsExpansion struct {
 	// The image that will run before the user service starts, to expand files artifacts into volumes
diff --git a/container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go b/container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go
new file mode 100644
index 000000000..725a86b83
--- /dev/null
+++ b/container-engine-lib/lib/backend_interface/objects/service_directory/persistent_directories.go
@@ -0,0 +1,7 @@
+package service_directory
+
+type DirectoryPersistentKey string
+
+type PersistentDirectories struct {
+	ServiceDirpathToDirectoryPersistentKey map[string]DirectoryPersistentKey
+}
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go
index 3c210e60f..aeab334fd 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go
@@ -174,6 +174,7 @@ func replaceMagicStrings(
 		cmdArgs,
 		envVars,
 		serviceConfig.GetFilesArtifactsExpansion(),
+		serviceConfig.GetPersistentDirectories(),
 		serviceConfig.GetCPUAllocationMillicpus(),
 		serviceConfig.GetMemoryAllocationMegabytes(),
 		serviceConfig.GetPrivateIPAddrPlaceholder(),
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go
index f7cebea83..e2b6715e8 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go
@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/shared_utils"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/files_artifacts_expansion"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_structure"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/magic_string_helper"
@@ -204,7 +204,7 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
 		image = defaultRunPythonImageName
 	}
 
-	var filesArtifactExpansion *files_artifacts_expansion.FilesArtifactsExpansion
+	var filesArtifactExpansion *service_directory.FilesArtifactsExpansion
 	if arguments.IsSet(FilesArgName) {
 		filesStarlark, err := builtin_argument.ExtractArgumentValue[*starlark.Dict](arguments, FilesArgName)
 		if err != nil {
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go
index 9113caf08..468b1f643 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go
@@ -3,8 +3,8 @@ package tasks
 import (
 	"context"
 	"fmt"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/files_artifacts_expansion"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/enclave_structure"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/magic_string_helper"
@@ -126,7 +126,7 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen
 		image = defaultRunShImageName
 	}
 
-	var filesArtifactExpansion *files_artifacts_expansion.FilesArtifactsExpansion
+	var filesArtifactExpansion *service_directory.FilesArtifactsExpansion
 	if arguments.IsSet(FilesArgName) {
 		filesStarlark, err := builtin_argument.ExtractArgumentValue[*starlark.Dict](arguments, FilesArgName)
 		if err != nil {
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go
index 5a3cf6c09..e288d4176 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go
@@ -4,8 +4,8 @@ import (
 	"context"
 	"fmt"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/files_artifacts_expansion"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers/magic_string_helper"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument"
@@ -229,7 +229,7 @@ func resultMapToString(resultMap map[string]starlark.Comparable, builtinNameForL
 	return fmt.Sprintf("Command returned with exit code '%v' and the following output: %v", exitCode, outputStr)
 }
 
-func getServiceConfig(image string, filesArtifactExpansion *files_artifacts_expansion.FilesArtifactsExpansion) *service.ServiceConfig {
+func getServiceConfig(image string, filesArtifactExpansion *service_directory.FilesArtifactsExpansion) *service.ServiceConfig {
 	return service.NewServiceConfig(
 		image,
 		nil,
@@ -243,6 +243,7 @@ func getServiceConfig(image string, filesArtifactExpansion *files_artifacts_expa
 		nil,
 		nil,
 		filesArtifactExpansion,
+		nil,
 		0,
 		0,
 		service_config.DefaultPrivateIPAddrPlaceholder,
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go
index 4583362e3..9793bb752 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go
@@ -2,9 +2,9 @@ package service_config
 
 import (
 	"fmt"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/files_artifacts_expansion"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
 	"github.com/kurtosis-tech/kurtosis/core/files_artifacts_expander/args"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework"
@@ -203,7 +203,7 @@ func (config *ServiceConfig) Copy() (builtin_argument.KurtosisValueType, error)
 	}, nil
 }
 
-func ConvertFilesArtifactsMounts(filesArtifactsMountDirpathsMap map[string]string, serviceNetwork service_network.ServiceNetwork) (*files_artifacts_expansion.FilesArtifactsExpansion, *startosis_errors.InterpretationError) {
+func ConvertFilesArtifactsMounts(filesArtifactsMountDirpathsMap map[string]string, serviceNetwork service_network.ServiceNetwork) (*service_directory.FilesArtifactsExpansion, *startosis_errors.InterpretationError) {
 	filesArtifactsExpansions := []args.FilesArtifactExpansion{}
 	serviceDirpathsToArtifactIdentifiers := map[string]string{}
 	expanderDirpathToUserServiceDirpathMap := map[string]string{}
@@ -242,7 +242,7 @@ func ConvertFilesArtifactsMounts(filesArtifactsMountDirpathsMap map[string]strin
 		apiContainerInfo.GetVersion(),
 	)
 
-	return &files_artifacts_expansion.FilesArtifactsExpansion{
+	return &service_directory.FilesArtifactsExpansion{
 		ExpanderImage:                        expanderImageAndTag,
 		ExpanderEnvVars:                      expanderEnvVars,
 		ServiceDirpathsToArtifactIdentifiers: serviceDirpathsToArtifactIdentifiers,
@@ -292,7 +292,7 @@ func (config *ServiceConfig) ToKurtosisType(serviceNetwork service_network.Servi
 		}
 	}
 
-	var filesArtifactExpansions *files_artifacts_expansion.FilesArtifactsExpansion
+	var filesArtifactExpansions *service_directory.FilesArtifactsExpansion
 	filesStarlark, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[*starlark.Dict](config.KurtosisValueTypeDefault, FilesAttr)
 	if interpretationErr != nil {
 		return nil, interpretationErr
@@ -451,6 +451,7 @@ func (config *ServiceConfig) ToKurtosisType(serviceNetwork service_network.Servi
 		cmdArgs,
 		envVars,
 		filesArtifactExpansions,
+		nil,
 		maxCpu,
 		maxMemory,
 		privateIpAddressPlaceholder,


Test patch:
diff --git a/core/server/api_container/server/service_network/default_service_network_test.go b/core/server/api_container/server/service_network/default_service_network_test.go
index c5a781f8d..e43ac8577 100644
--- a/core/server/api_container/server/service_network/default_service_network_test.go
+++ b/core/server/api_container/server/service_network/default_service_network_test.go
@@ -2112,6 +2112,7 @@ func testServiceConfig(imageName string, subnetwork string) *service.ServiceConf
 		nil,
 		nil,
 		nil,
+		nil,
 		0,
 		0,
 		"",
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared_test.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared_test.go
index 3748ad0fd..7b3ce99cb 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared_test.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared_test.go
@@ -33,6 +33,7 @@ func TestAddServiceShared_EntryPointArgsRuntimeValueAreReplaced(t *testing.T) {
 		nil,
 		nil,
 		nil,
+		nil,
 		0,
 		0,
 		"",
@@ -66,6 +67,7 @@ func TestAddServiceShared_CmdArgsRuntimeValueAreReplaced(t *testing.T) {
 		[]string{"bash", "-c", "sleep " + runtimeValue},
 		nil,
 		nil,
+		nil,
 		0,
 		0,
 		"",
@@ -101,6 +103,7 @@ func TestAddServiceShared_EnvVarsWithRuntimeValueAreReplaced(t *testing.T) {
 			"PORT": runtimeValue,
 		},
 		nil,
+		nil,
 		0,
 		0,
 		"",
@@ -137,6 +140,7 @@ func TestAddServiceShared_ServiceNameWithRuntimeValuesAreReplaced(t *testing.T)
 		nil,
 		nil,
 		nil,
+		nil,
 		0,
 		0,
 		"",
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go
index 6c8731b2e..ab99e3ca2 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go
@@ -49,6 +49,7 @@ func (t *addServiceTestCase) GetInstruction() *kurtosis_plan_instruction.Kurtosi
 				nil,
 				map[string]string{},
 				nil,
+				nil,
 				0,
 				0,
 				service_config.DefaultPrivateIPAddrPlaceholder,
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_services_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_services_framework_test.go
index 1b4d17f87..0d26615ac 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_services_framework_test.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_services_framework_test.go
@@ -65,6 +65,7 @@ func (t *addServicesTestCase) GetInstruction() *kurtosis_plan_instruction.Kurtos
 				nil,
 				map[string]string{},
 				nil,
+				nil,
 				0,
 				0,
 				service_config.DefaultPrivateIPAddrPlaceholder,
@@ -83,6 +84,7 @@ func (t *addServicesTestCase) GetInstruction() *kurtosis_plan_instruction.Kurtos
 				nil,
 				map[string]string{},
 				nil,
+				nil,
 				TestCpuAllocation,
 				TestMemoryAllocation,
 				service_config.DefaultPrivateIPAddrPlaceholder,
diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_minimal_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_minimal_framework_test.go
index 7f69e0bb6..696963020 100644
--- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_minimal_framework_test.go
+++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_minimal_framework_test.go
@@ -49,6 +49,7 @@ func (t *serviceConfigMinimalTestCase) Assert(typeValue builtin_argument.Kurtosi
 		nil,
 		map[string]string{},
 		nil,
+		nil,
 		0,
 		0,
 		service_config.DefaultPrivateIPAddrPlaceholder,