Repository: viamrobotics/rdk
Pull request #4914: DATA-3962 Make file deletion and capture dir threshold tunable
Author: login: vijayvuyyuru
Labels: name: safe to test
description: This pull request is marked safe to test from a trusted zone
Created: 2025-04-15T20:10:08Z | Merged: 2025-04-18T19:15:59Z

Hey yall, this PR adds the file deletion threshold and capture dir usage ratio as config params instead of being hardcoded.
If unset, the default values are unchanged. They are >90% disk usage and >50% is the capture dir. 
An example config using these values could look like this:
```json5
{
  "sync_interval_mins": 0.1,
  "capture_dir": "",
  "tags": [],
  "additional_sync_paths": [],
  "disk_usage_deletion_threshold": 0.9,
  "capture_dir_deletion_threshold": 0.9
}
```

### Testing:
Added a debug log to the function that runs on an interval to check for file deletion. Verified that any time I reconfigured my sync config with new values, I saw them in the next log line. 


Diff:
diff --git a/services/datamanager/builtin/builtin_test.go b/services/datamanager/builtin/builtin_test.go
index 2569d3f5aa9..1b65ccc1ce4 100644
--- a/services/datamanager/builtin/builtin_test.go
+++ b/services/datamanager/builtin/builtin_test.go
@@ -262,17 +262,10 @@ func TestFileDeletion(t *testing.T) {
 	tempDir := t.TempDir()
 	ctx := context.Background()
 
-	fsThresholdToTriggerDeletion := datasync.FSThresholdToTriggerDeletion
-	captureDirToFSUsageRatio := datasync.CaptureDirToFSUsageRatio
 	t.Cleanup(func() {
 		clk = prevClock
-		datasync.FSThresholdToTriggerDeletion = fsThresholdToTriggerDeletion
-		datasync.CaptureDirToFSUsageRatio = captureDirToFSUsageRatio
 	})
 
-	datasync.FSThresholdToTriggerDeletion = math.SmallestNonzeroFloat64
-	datasync.CaptureDirToFSUsageRatio = math.SmallestNonzeroFloat64
-
 	r := setupRobot(nil, map[resource.Name]resource.Resource{
 		arm.Named("arm1"): &inject.Arm{
 			EndPositionFunc: func(
@@ -315,6 +308,8 @@ func TestFileDeletion(t *testing.T) {
 	// MaximumCaptureFileSizeBytes is set to 1 so that each reading becomes its own capture file
 	// and we can confidently read the capture file without it's contents being modified by the collector
 	c.MaximumCaptureFileSizeBytes = 1
+	c.DiskUsageDeletionThreshold = math.SmallestNonzeroFloat64
+	c.CaptureDirDeletionThreshold = math.SmallestNonzeroFloat64
 	bSvc, err := New(ctx, deps, config, datasync.NoOpCloudClientConstructor, connToConnectivityStateError, logger)
 	test.That(t, err, test.ShouldBeNil)
 	b := bSvc.(*builtIn)
diff --git a/services/datamanager/builtin/config.go b/services/datamanager/builtin/config.go
index f8be2dd094d..277b382246b 100644
--- a/services/datamanager/builtin/config.go
+++ b/services/datamanager/builtin/config.go
@@ -24,6 +24,12 @@ const (
 	// which is evaluated if the file deletion threshold has been reached. If `captureFileIndex % N == 0`
 	// return true then the file will be deleted to free up space.
 	defaultDeleteEveryNth = 5
+	// defaultDiskUsageThreshold and defaultCaptureDirThreshold are the thresholds at which file deletion might occur.
+	// If disk usage is at or above this threshold, AND the capture directory makes up at least CaptureDirThreshold (%) of the disk usage,
+	// then file deletion will occur. If disk usage is at or above the disk usage threshold, but the capture directory is
+	// below the capture directory threshold, then file deletion will not occur but a warning will be logged periodically.
+	defaultDiskUsageThreshold  = 0.9
+	defaultCaptureDirThreshold = 0.5
 	// defaultSyncIntervalMins is the sync interval that will be set if the config's sync_interval_mins is zero (including when it is unset).
 	defaultSyncIntervalMins = 0.1
 	// syncIntervalMinsEpsilon is the value below which SyncIntervalMins is considered zero.
@@ -42,10 +48,13 @@ type Config struct {
 	CaptureDir string   `json:"capture_dir"`
 	Tags       []string `json:"tags"`
 	// Capture
-	CaptureDisabled             bool                 `json:"capture_disabled"`
-	DeleteEveryNthWhenDiskFull  int                  `json:"delete_every_nth_when_disk_full"`
-	MaximumCaptureFileSizeBytes int64                `json:"maximum_capture_file_size_bytes"`
-	MongoCaptureConfig          *capture.MongoConfig `json:"mongo_capture_config"`
+	CaptureDisabled    bool                 `json:"capture_disabled"`
+	MongoCaptureConfig *capture.MongoConfig `json:"mongo_capture_config"`
+	// File Deletion Parameters
+	DeleteEveryNthWhenDiskFull  int     `json:"delete_every_nth_when_disk_full"`
+	MaximumCaptureFileSizeBytes int64   `json:"maximum_capture_file_size_bytes"`
+	DiskUsageDeletionThreshold  float64 `json:"disk_usage_deletion_threshold"`
+	CaptureDirDeletionThreshold float64 `json:"capture_dir_deletion_threshold"`
 	// Sync
 	AdditionalSyncPaths    []string `json:"additional_sync_paths"`
 	FileLastModifiedMillis int      `json:"file_last_modified_millis"`
@@ -72,6 +81,12 @@ func (c *Config) Validate(path string) ([]string, error) {
 	if c.DeleteEveryNthWhenDiskFull < 0 {
 		return nil, errors.New("delete_every_nth_when_disk_full can't be negative")
 	}
+	if c.DiskUsageDeletionThreshold < 0 {
+		return nil, errors.New("disk_usage_deletion_threshold can't be negative")
+	}
+	if c.CaptureDirDeletionThreshold < 0 {
+		return nil, errors.New("capture_dir_deletion_threshold can't be negative")
+	}
 	return []string{cloud.InternalServiceName.String()}, nil
 }
 
@@ -118,6 +133,18 @@ func (c *Config) syncConfig(syncSensor sensor.Sensor, syncSensorEnabled bool, lo
 	}
 	c.DeleteEveryNthWhenDiskFull = deleteEveryNthValue
 
+	diskUsageThreshold := defaultDiskUsageThreshold
+	if c.DiskUsageDeletionThreshold != 0 {
+		diskUsageThreshold = c.DiskUsageDeletionThreshold
+	}
+	c.DiskUsageDeletionThreshold = diskUsageThreshold
+
+	captureDirThreshold := defaultCaptureDirThreshold
+	if c.CaptureDirDeletionThreshold != 0 {
+		captureDirThreshold = c.CaptureDirDeletionThreshold
+	}
+	c.CaptureDirDeletionThreshold = captureDirThreshold
+
 	fileLastModifiedMillis := c.FileLastModifiedMillis
 	if fileLastModifiedMillis <= 0 {
 		fileLastModifiedMillis = defaultFileLastModifiedMillis
@@ -132,17 +159,19 @@ func (c *Config) syncConfig(syncSensor sensor.Sensor, syncSensorEnabled bool, lo
 	}
 
 	return datasync.Config{
-		AdditionalSyncPaths:        c.AdditionalSyncPaths,
-		Tags:                       c.Tags,
-		CaptureDir:                 c.getCaptureDir(logger),
-		CaptureDisabled:            c.CaptureDisabled,
-		DeleteEveryNthWhenDiskFull: c.DeleteEveryNthWhenDiskFull,
-		FileLastModifiedMillis:     c.FileLastModifiedMillis,
-		MaximumNumSyncThreads:      c.MaximumNumSyncThreads,
-		ScheduledSyncDisabled:      c.ScheduledSyncDisabled,
-		SelectiveSyncerName:        c.SelectiveSyncerName,
-		SyncIntervalMins:           syncIntervalMins,
-		SelectiveSyncSensor:        syncSensor,
-		SelectiveSyncSensorEnabled: syncSensorEnabled,
+		AdditionalSyncPaths:         c.AdditionalSyncPaths,
+		Tags:                        c.Tags,
+		CaptureDir:                  c.getCaptureDir(logger),
+		CaptureDisabled:             c.CaptureDisabled,
+		DeleteEveryNthWhenDiskFull:  c.DeleteEveryNthWhenDiskFull,
+		DiskUsageDeletionThreshold:  c.DiskUsageDeletionThreshold,
+		CaptureDirDeletionThreshold: c.CaptureDirDeletionThreshold,
+		FileLastModifiedMillis:      c.FileLastModifiedMillis,
+		MaximumNumSyncThreads:       c.MaximumNumSyncThreads,
+		ScheduledSyncDisabled:       c.ScheduledSyncDisabled,
+		SelectiveSyncerName:         c.SelectiveSyncerName,
+		SyncIntervalMins:            syncIntervalMins,
+		SelectiveSyncSensor:         syncSensor,
+		SelectiveSyncSensorEnabled:  syncSensorEnabled,
 	}
 }
diff --git a/services/datamanager/builtin/config_test.go b/services/datamanager/builtin/config_test.go
index a09f21caa00..6fa4f1e8aef 100644
--- a/services/datamanager/builtin/config_test.go
+++ b/services/datamanager/builtin/config_test.go
@@ -28,6 +28,8 @@ var fullConfig = &Config{
 	SelectiveSyncerName:         "some name",
 	SyncIntervalMins:            0.5,
 	Tags:                        []string{"a", "b", "c"},
+	DiskUsageDeletionThreshold:  0.1,
+	CaptureDirDeletionThreshold: 0.1,
 }
 
 func TestConfig(t *testing.T) {
@@ -71,6 +73,16 @@ func TestConfig(t *testing.T) {
 				config: Config{DeleteEveryNthWhenDiskFull: -1},
 				err:    errors.New("delete_every_nth_when_disk_full can't be negative"),
 			},
+			{
+				name:   "returns an error if DiskUsageThreshold is negative",
+				config: Config{DiskUsageDeletionThreshold: -1},
+				err:    errors.New("disk_usage_deletion_threshold can't be negative"),
+			},
+			{
+				name:   "returns an error if CaptureDirDeletionThreshold is negative",
+				config: Config{CaptureDirDeletionThreshold: -1},
+				err:    errors.New("capture_dir_deletion_threshold can't be negative"),
+			},
 		}
 
 		for _, tc := range tcs {
@@ -128,39 +140,45 @@ func TestConfig(t *testing.T) {
 		t.Run("returns a sync config with defaults when called on an empty config", func(t *testing.T) {
 			c := &Config{}
 			test.That(t, c.syncConfig(nil, false, logger), test.ShouldResemble, sync.Config{
-				CaptureDir:                 viamCaptureDotDir,
-				DeleteEveryNthWhenDiskFull: 5,
-				FileLastModifiedMillis:     10000,
-				MaximumNumSyncThreads:      runtime.NumCPU() / 2,
-				SyncIntervalMins:           0.1,
+				CaptureDir:                  viamCaptureDotDir,
+				DeleteEveryNthWhenDiskFull:  5,
+				FileLastModifiedMillis:      10000,
+				MaximumNumSyncThreads:       runtime.NumCPU() / 2,
+				SyncIntervalMins:            0.1,
+				DiskUsageDeletionThreshold:  0.9,
+				CaptureDirDeletionThreshold: 0.5,
 			})
 		})
 
 		t.Run("returns a sync config with defaults when called on a config with SyncIntervalMins which is practically 0", func(t *testing.T) {
 			c := &Config{SyncIntervalMins: 0.000000000000000001}
 			test.That(t, c.syncConfig(nil, false, logger), test.ShouldResemble, sync.Config{
-				CaptureDir:                 viamCaptureDotDir,
-				DeleteEveryNthWhenDiskFull: 5,
-				FileLastModifiedMillis:     10000,
-				MaximumNumSyncThreads:      runtime.NumCPU() / 2,
-				SyncIntervalMins:           0.1,
+				CaptureDir:                  viamCaptureDotDir,
+				DeleteEveryNthWhenDiskFull:  5,
+				FileLastModifiedMillis:      10000,
+				MaximumNumSyncThreads:       runtime.NumCPU() / 2,
+				SyncIntervalMins:            0.1,
+				DiskUsageDeletionThreshold:  0.9,
+				CaptureDirDeletionThreshold: 0.5,
 			})
 		})
 		t.Run("returns a sync config with overridden defaults when called on a full config", func(t *testing.T) {
 			s := &inject.Sensor{}
 			test.That(t, fullConfig.syncConfig(s, true, logger), test.ShouldResemble, sync.Config{
-				AdditionalSyncPaths:        []string{"/tmp/a", "/tmp/b"},
-				CaptureDir:                 "/tmp/some/path",
-				CaptureDisabled:            true,
-				DeleteEveryNthWhenDiskFull: 2,
-				FileLastModifiedMillis:     50000,
-				MaximumNumSyncThreads:      10,
-				ScheduledSyncDisabled:      true,
-				SelectiveSyncSensor:        s,
-				SelectiveSyncSensorEnabled: true,
-				SelectiveSyncerName:        "some name",
-				SyncIntervalMins:           0.5,
-				Tags:                       []string{"a", "b", "c"},
+				AdditionalSyncPaths:         []string{"/tmp/a", "/tmp/b"},
+				CaptureDir:                  "/tmp/some/path",
+				CaptureDisabled:             true,
+				DeleteEveryNthWhenDiskFull:  2,
+				DiskUsageDeletionThreshold:  0.1,
+				CaptureDirDeletionThreshold: 0.1,
+				FileLastModifiedMillis:      50000,
+				MaximumNumSyncThreads:       10,
+				ScheduledSyncDisabled:       true,
+				SelectiveSyncSensor:         s,
+				SelectiveSyncSensorEnabled:  true,
+				SelectiveSyncerName:         "some name",
+				SyncIntervalMins:            0.5,
+				Tags:                        []string{"a", "b", "c"},
 			})
 		})
 	})
diff --git a/services/datamanager/builtin/sync/config.go b/services/datamanager/builtin/sync/config.go
index 833bce9622f..6e96293884d 100644
--- a/services/datamanager/builtin/sync/config.go
+++ b/services/datamanager/builtin/sync/config.go
@@ -49,8 +49,8 @@ type Config struct {
 	// which in english reads:
 	//
 	// If datacapture is enabled then every 30 seconds
-	// if the disk is full (which is defined as the disk
-	// is 90% full and 50% is contributed by the CaptureDir)
+	// if the disk is full (file system usage is at or above DiskUsageDeletionThreshold
+	// and CaptureDirToFSThreshold of the disk usage is contributed by the CaptureDir)
 	// delete every Nth file in the CaptureDir & child
 	// directories.
 	//
@@ -58,6 +58,15 @@ type Config struct {
 	// disk if the robot is unable to sync data for a long period
 	// of time. Defaults to 5.
 	DeleteEveryNthWhenDiskFull int
+	// DiskUsageDeletionThreshold defines the threshold at which file deletion might occur.
+	// If disk usage is at or above this threshold, AND the capture directory makes up at least CaptureDirToFSThreshold of the disk usage,
+	// then file deletion will occur based on the DeleteEveryNthWhenDiskFull parameter. If disk usage is at or above the disk usage threshold,
+	// but the capture directory is below the capture directory threshold, then file deletion will not occur but a
+	// warning will be logged periodically.
+	// Defaults to 0.90.
+	DiskUsageDeletionThreshold float64
+	// Defaults to 0.50
+	CaptureDirDeletionThreshold float64
 	// FileLastModifiedMillis defines the number of milliseconds that
 	// we should wait for an arbitrary file (aka a file that doesn't end in
 	// either the .prog nor the .capture file extension) before we consider
@@ -99,6 +108,8 @@ func (c Config) Equal(o Config) bool {
 		c.CaptureDir == o.CaptureDir &&
 		c.CaptureDisabled == o.CaptureDisabled &&
 		c.DeleteEveryNthWhenDiskFull == o.DeleteEveryNthWhenDiskFull &&
+		c.DiskUsageDeletionThreshold == o.DiskUsageDeletionThreshold &&
+		c.CaptureDirDeletionThreshold == o.CaptureDirDeletionThreshold &&
 		c.FileLastModifiedMillis == o.FileLastModifiedMillis &&
 		c.MaximumNumSyncThreads == o.MaximumNumSyncThreads &&
 		c.ScheduledSyncDisabled == o.ScheduledSyncDisabled &&
diff --git a/services/datamanager/builtin/sync/config_test.go b/services/datamanager/builtin/sync/config_test.go
index f2eed96ae2a..d7dbac3604c 100644
--- a/services/datamanager/builtin/sync/config_test.go
+++ b/services/datamanager/builtin/sync/config_test.go
@@ -28,32 +28,36 @@ func TestConfig(t *testing.T) {
 			{
 				name: "full configs that are equal are equal",
 				a: Config{
-					AdditionalSyncPaths:        []string{"a", "b"},
-					CaptureDir:                 "c",
-					CaptureDisabled:            true,
-					DeleteEveryNthWhenDiskFull: 5,
-					FileLastModifiedMillis:     100,
-					MaximumNumSyncThreads:      100,
-					ScheduledSyncDisabled:      true,
-					SelectiveSyncerName:        "tom",
-					SyncIntervalMins:           1.1,
-					Tags:                       []string{"c", "d"},
-					SelectiveSyncSensorEnabled: true,
-					SelectiveSyncSensor:        sensorA,
+					AdditionalSyncPaths:         []string{"a", "b"},
+					CaptureDir:                  "c",
+					CaptureDisabled:             true,
+					DeleteEveryNthWhenDiskFull:  5,
+					DiskUsageDeletionThreshold:  0.90,
+					CaptureDirDeletionThreshold: 0.5,
+					FileLastModifiedMillis:      100,
+					MaximumNumSyncThreads:       100,
+					ScheduledSyncDisabled:       true,
+					SelectiveSyncerName:         "tom",
+					SyncIntervalMins:            1.1,
+					Tags:                        []string{"c", "d"},
+					SelectiveSyncSensorEnabled:  true,
+					SelectiveSyncSensor:         sensorA,
 				},
 				b: Config{
-					AdditionalSyncPaths:        []string{"a", "b"},
-					CaptureDir:                 "c",
-					CaptureDisabled:            true,
-					DeleteEveryNthWhenDiskFull: 5,
-					FileLastModifiedMillis:     100,
-					MaximumNumSyncThreads:      100,
-					ScheduledSyncDisabled:      true,
-					SelectiveSyncerName:        "tom",
-					SyncIntervalMins:           1.1,
-					Tags:                       []string{"c", "d"},
-					SelectiveSyncSensorEnabled: true,
-					SelectiveSyncSensor:        sensorA,
+					AdditionalSyncPaths:         []string{"a", "b"},
+					CaptureDir:                  "c",
+					CaptureDisabled:             true,
+					DeleteEveryNthWhenDiskFull:  5,
+					DiskUsageDeletionThreshold:  0.90,
+					CaptureDirDeletionThreshold: 0.5,
+					FileLastModifiedMillis:      100,
+					MaximumNumSyncThreads:       100,
+					ScheduledSyncDisabled:       true,
+					SelectiveSyncerName:         "tom",
+					SyncIntervalMins:            1.1,
+					Tags:                        []string{"c", "d"},
+					SelectiveSyncSensorEnabled:  true,
+					SelectiveSyncSensor:         sensorA,
 				},
 				equal: true,
 			},
@@ -97,6 +101,26 @@ func TestConfig(t *testing.T) {
 				},
 				equal: false,
 			},
+			{
+				name: "different DiskUsageThreshold are not equal",
+				a: Config{
+					DiskUsageDeletionThreshold: 0.90,
+				},
+				b: Config{
+					DiskUsageDeletionThreshold: 0.80,
+				},
+				equal: false,
+			},
+			{
+				name: "different CaptureDirThreshold are not equal",
+				a: Config{
+					CaptureDirDeletionThreshold: 0.5,
+				},
+				b: Config{
+					CaptureDirDeletionThreshold: 0.6,
+				},
+				equal: false,
+			},
 			{
 				name: "different FileLastModifiedMillis are not equal",
 				a: Config{
diff --git a/services/datamanager/builtin/sync/file_deletion.go b/services/datamanager/builtin/sync/file_deletion.go
index ec3bb0afa7b..0a4bd121ad1 100644
--- a/services/datamanager/builtin/sync/file_deletion.go
+++ b/services/datamanager/builtin/sync/file_deletion.go
@@ -17,13 +17,6 @@ import (
 	"go.viam.com/rdk/utils/diskusage"
 )
 
-var (
-	// FSThresholdToTriggerDeletion temporarily public for tests.
-	FSThresholdToTriggerDeletion = .90
-	// CaptureDirToFSUsageRatio temporarily public for tests.
-	CaptureDirToFSUsageRatio = .5
-)
-
 var errAtSizeThreshold = errors.New("capture directory has reached or exceeded disk usage threshold for deletion")
 
 func deleteExcessFilesOnSchedule(
@@ -31,6 +24,8 @@ func deleteExcessFilesOnSchedule(
 	fileTracker *fileTracker,
 	captureDir string,
 	deleteEveryNth int,
+	diskUsageThreshold float64,
+	captureDirThreshold float64,
 	clock clock.Clock,
 	logger logging.Logger,
 ) {
@@ -49,7 +44,7 @@ func deleteExcessFilesOnSchedule(
 		case <-ctx.Done():
 			return
 		case <-t.C:
-			maybeDeleteExcessFiles(ctx, fileTracker, captureDir, deleteEveryNth, clock, logger)
+			maybeDeleteExcessFiles(ctx, fileTracker, captureDir, deleteEveryNth, diskUsageThreshold, captureDirThreshold, clock, logger)
 		}
 	}
 }
@@ -59,6 +54,8 @@ func maybeDeleteExcessFiles(
 	fileTracker *fileTracker,
 	captureDir string,
 	deleteEveryNth int,
+	diskUsageThreshold float64,
+	captureDirThreshold float64,
 	clock clock.Clock,
 	logger logging.Logger,
 ) {
@@ -81,6 +78,8 @@ func maybeDeleteExcessFiles(
 		usage,
 		captureDir,
 		deleteEveryNth,
+		diskUsageThreshold,
+		captureDirThreshold,
 		logger)
 
 	duration := clock.Since(start)
@@ -101,12 +100,16 @@ func deleteExcessFiles(
 	usage diskusage.DiskUsage,
 	captureDir string,
 	deleteEveryNth int,
+	diskUsageThreshold float64,
+	captureDirToFSThreshold float64,
 	logger logging.Logger,
 ) (int, error) {
 	shouldDelete, err := shouldDeleteBasedOnDiskUsage(
 		ctx,
 		usage,
 		captureDir,
+		diskUsageThreshold,
+		captureDirToFSThreshold,
 		logger)
 	if err != nil {
 		return 0, errors.Wrap(err, "error checking file system stats")
@@ -116,7 +119,7 @@ func deleteExcessFiles(
 		return 0, nil
 	}
 
-	logger.Warnf("current disk usage of the data capture directory exceeds threshold (%f)", CaptureDirToFSUsageRatio)
+	logger.Warnf("current disk usage of the data capture directory exceeds threshold (%f)", captureDirToFSThreshold)
 	return deleteFiles(ctx, fileTracker, deleteEveryNth, captureDir, logger)
 }
 
@@ -124,12 +127,14 @@ func shouldDeleteBasedOnDiskUsage(
 	ctx context.Context,
 	usage diskusage.DiskUsage,
 	captureDirPath string,
+	diskUsageThreshold float64,
+	captureDirToFSThreshold float64,
 	logger logging.Logger,
 ) (bool, error) {
 	usedSpace := 1.0 - usage.AvailablePercent()
-	if usedSpace < FSThresholdToTriggerDeletion {
+	if usedSpace < diskUsageThreshold {
 		logger.Debugf("disk not full enough. Threshold: %s, Used space: %s, %s",
-			fmt.Sprintf("%.2f", FSThresholdToTriggerDeletion*100)+"%",
+			fmt.Sprintf("%.2f", diskUsageThreshold*100)+"%",
 			fmt.Sprintf("%.2f", usedSpace*100)+"%",
 			usage)
 		return false, nil
@@ -139,11 +144,11 @@ func shouldDeleteBasedOnDiskUsage(
 		ctx,
 		captureDirPath,
 		float64(usage.SizeBytes),
-		CaptureDirToFSUsageRatio,
+		captureDirToFSThreshold,
 	)
 	if !shouldDelete {
 		logger.Warnf("Disk nearing capacity but data capture directory is below %f of that size, file deletion will not run",
-			CaptureDirToFSUsageRatio)
+			captureDirToFSThreshold)
 	}
 	return shouldDelete, err
 }
diff --git a/services/datamanager/builtin/sync/file_deletion_test.go b/services/datamanager/builtin/sync/file_deletion_test.go
index e4ddbdb60fb..3dbffafa4da 100644
--- a/services/datamanager/builtin/sync/file_deletion_test.go
+++ b/services/datamanager/builtin/sync/file_deletion_test.go
@@ -123,19 +123,11 @@ func TestFileDeletionUsageCheck(t *testing.T) {
 			tempCaptureDir := t.TempDir()
 			// write testing files
 			writeFiles(t, tempCaptureDir, []string{"1.capture", "2.capture"})
-			// overwrite thresholds
-			fsThresholdToTriggerDeletion := FSThresholdToTriggerDeletion
-			captureDirToFSUsageRatio := CaptureDirToFSUsageRatio
-			FSThresholdToTriggerDeletion = tc.triggerThreshold
-			CaptureDirToFSUsageRatio = tc.captureUsageRatio
-			t.Cleanup(func() {
-				FSThresholdToTriggerDeletion = fsThresholdToTriggerDeletion
-				CaptureDirToFSUsageRatio = captureDirToFSUsageRatio
-			})
 			logger := logging.NewTestLogger(t)
 			usage, err := diskusage.Statfs(tempCaptureDir)
 			test.That(t, err, test.ShouldBeNil)
-			willDelete, err := shouldDeleteBasedOnDiskUsage(context.Background(), usage, tempCaptureDir, logger)
+			willDelete, err := shouldDeleteBasedOnDiskUsage(context.Background(), usage, tempCaptureDir,
+				tc.triggerThreshold, tc.captureUsageRatio, logger)
 			test.That(t, err, test.ShouldBeNil)
 			test.That(t, willDelete, test.ShouldEqual, tc.deletionExpected)
 		})
diff --git a/services/datamanager/builtin/sync/sync.go b/services/datamanager/builtin/sync/sync.go
index 29d1759f888..084cedded90 100644
--- a/services/datamanager/builtin/sync/sync.go
+++ b/services/datamanager/builtin/sync/sync.go
@@ -180,6 +180,8 @@ func (s *Sync) Reconfigure(_ context.Context, config Config, cloudConnSvc cloud.
 				s.fileTracker,
 				config.CaptureDir,
 				config.DeleteEveryNthWhenDiskFull,
+				config.DiskUsageDeletionThreshold,
+				config.CaptureDirDeletionThreshold,
 				s.clock,
 				s.logger,
 			)
diff --git a/services/datamanager/data/fake_robot_with_many_collectors_data_manager.json b/services/datamanager/data/fake_robot_with_many_collectors_data_manager.json
index 16289c1e508..003a006007c 100644
--- a/services/datamanager/data/fake_robot_with_many_collectors_data_manager.json
+++ b/services/datamanager/data/fake_robot_with_many_collectors_data_manager.json
@@ -66,7 +66,9 @@
                 "sync_disabled": false,
                 "sync_interval_mins": 0,
                 "capture_dir": "/tmp/capture",
-                "capture_disabled": false
+                "capture_disabled": false,
+                "disk_usage_threshold_percent": 0.00000000000000001,
+                "capture_dir_threshold_percent": 0.00000000000000001
             }
         }
     ]


Reviews:
user: login: katiepeters
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T14:55:57Z
user: login: vijayvuyyuru
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T15:14:30Z
user: login: vijayvuyyuru
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T15:14:47Z
user: login: vijayvuyyuru
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T15:18:08Z
user: login: vijayvuyyuru
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T15:19:51Z
user: login: katiepeters
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-18T18:17:14Z
user: login: katiepeters
body: wahoo! let's delete some stuff!!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-18T19:14:05Z

Review comments:
diff_hunk: @@ -58,6 +58,15 @@ type Config struct {
 	// disk if the robot is unable to sync data for a long period
 	// of time. Defaults to 5.
 	DeleteEveryNthWhenDiskFull int
+	// DiskUsageDeletionThreshold defines the threshold at which file deletion might occur.
+	// If disk usage is at or above this threshold, AND the capture directory makes up at least CaptureDirToFSThreshold of the disk usage,
+	// then file deletion will occur based on the DeleteEveryNthWhenDiskFull parameter. If disk usage is at or above the disk usage threshold,
+	// but the capture directory is below the capture directory threshold, then file deletion will not occur but a
+	// warning will be logged periodically.
+	// Defaults to 0.90.
path: services/datamanager/builtin/sync/config.go
user: login: katiepeters
body: Nit: Maybe adjust the comments for `DeleteEveryNthWhenDiskFull`
created_at: 2025-04-17T14:43:40Z
author_association: MEMBER
diff_hunk: @@ -24,6 +24,12 @@ const (
 	// which is evaluated if the file deletion threshold has been reached. If `captureFileIndex % N == 0`
 	// return true then the file will be deleted to free up space.
 	defaultDeleteEveryNth = 5
+	// defaultDiskUsageThresholdPercent and defaultCaptureDirThresholdPercent are the thresholds at which file deletion might occur.
+	// If disk usage is at or above this threshold, AND the capture directory makes up at least CaptureDirThresholdPercent of the disk usage,
+	// then file deletion will occur. If disk usage is at or above the disk usage threshold, but the capture directory is
+	// below the capture directory threshold, then file deletion will not occur but a warning will be logged periodically.
+	defaultDiskUsageThresholdPercent  = 0.90
path: services/datamanager/builtin/config.go
user: login: katiepeters
body: [Question]: Why 0.90 and not 0.9?
created_at: 2025-04-17T14:44:24Z
author_association: MEMBER
diff_hunk: @@ -118,6 +133,18 @@ func (c *Config) syncConfig(syncSensor sensor.Sensor, syncSensorEnabled bool, lo
 	}
 	c.DeleteEveryNthWhenDiskFull = deleteEveryNthValue
 
+	diskUsageThresholdPercent := defaultDiskUsageThresholdPercent
path: services/datamanager/builtin/config.go
user: login: katiepeters
body: Question: Is it confusing to sometimes just say "threshold" but other times "threshold percent"?
created_at: 2025-04-17T14:51:11Z
author_association: MEMBER
diff_hunk: @@ -137,6 +164,8 @@ func (c *Config) syncConfig(syncSensor sensor.Sensor, syncSensorEnabled bool, lo
 		CaptureDir:                 c.getCaptureDir(logger),
 		CaptureDisabled:            c.CaptureDisabled,
 		DeleteEveryNthWhenDiskFull: c.DeleteEveryNthWhenDiskFull,
+		DiskUsageDeletionThreshold: c.DiskUsageDeletionThreshold,
+		CaptureDirToFSThreshold:    c.CaptureDirDeletionThreshold,
path: services/datamanager/builtin/config.go
user: login: katiepeters
body: Question: Are we changing the name again? A little confusing to keep track of
created_at: 2025-04-17T14:53:08Z
author_association: MEMBER
diff_hunk: @@ -24,6 +24,12 @@ const (
 	// which is evaluated if the file deletion threshold has been reached. If `captureFileIndex % N == 0`
 	// return true then the file will be deleted to free up space.
 	defaultDeleteEveryNth = 5
+	// defaultDiskUsageThresholdPercent and defaultCaptureDirThresholdPercent are the thresholds at which file deletion might occur.
+	// If disk usage is at or above this threshold, AND the capture directory makes up at least CaptureDirThresholdPercent of the disk usage,
+	// then file deletion will occur. If disk usage is at or above the disk usage threshold, but the capture directory is
+	// below the capture directory threshold, then file deletion will not occur but a warning will be logged periodically.
+	defaultDiskUsageThresholdPercent  = 0.90
path: services/datamanager/builtin/config.go
user: login: vijayvuyyuru
body: No particular reason. Removed the 0
created_at: 2025-04-17T15:14:29Z
author_association: MEMBER
diff_hunk: @@ -118,6 +133,18 @@ func (c *Config) syncConfig(syncSensor sensor.Sensor, syncSensorEnabled bool, lo
 	}
 	c.DeleteEveryNthWhenDiskFull = deleteEveryNthValue
 
+	diskUsageThresholdPercent := defaultDiskUsageThresholdPercent
path: services/datamanager/builtin/config.go
user: login: vijayvuyyuru
body: Sorry, couldnt decide on variable naming and thought I cleaned it up. 
created_at: 2025-04-17T15:14:47Z
author_association: MEMBER
diff_hunk: @@ -137,6 +164,8 @@ func (c *Config) syncConfig(syncSensor sensor.Sensor, syncSensorEnabled bool, lo
 		CaptureDir:                 c.getCaptureDir(logger),
 		CaptureDisabled:            c.CaptureDisabled,
 		DeleteEveryNthWhenDiskFull: c.DeleteEveryNthWhenDiskFull,
+		DiskUsageDeletionThreshold: c.DiskUsageDeletionThreshold,
+		CaptureDirToFSThreshold:    c.CaptureDirDeletionThreshold,
path: services/datamanager/builtin/config.go
user: login: vijayvuyyuru
body: So this one I wanted a different name so that it was clearer when referencing it in the actual deletion code what that value actually meant. If you think the name change is more confusing, I can keep the name as is and let the comment fill in that context.
created_at: 2025-04-17T15:18:08Z
author_association: MEMBER
diff_hunk: @@ -58,6 +58,15 @@ type Config struct {
 	// disk if the robot is unable to sync data for a long period
 	// of time. Defaults to 5.
 	DeleteEveryNthWhenDiskFull int
+	// DiskUsageDeletionThreshold defines the threshold at which file deletion might occur.
+	// If disk usage is at or above this threshold, AND the capture directory makes up at least CaptureDirToFSThreshold of the disk usage,
+	// then file deletion will occur based on the DeleteEveryNthWhenDiskFull parameter. If disk usage is at or above the disk usage threshold,
+	// but the capture directory is below the capture directory threshold, then file deletion will not occur but a
+	// warning will be logged periodically.
+	// Defaults to 0.90.
path: services/datamanager/builtin/sync/config.go
user: login: vijayvuyyuru
body: Good call
created_at: 2025-04-17T15:19:51Z
author_association: MEMBER
diff_hunk: @@ -24,6 +24,12 @@ const (
 	// which is evaluated if the file deletion threshold has been reached. If `captureFileIndex % N == 0`
 	// return true then the file will be deleted to free up space.
 	defaultDeleteEveryNth = 5
+	// defaultDiskUsageThreshold and defaultCaptureDirThresholdPercent are the thresholds at which file deletion might occur.
path: services/datamanager/builtin/config.go
user: login: katiepeters
body: ```suggestion
	// defaultDiskUsageThreshold and defaultCaptureDirThreshold are the thresholds at which file deletion might occur.
```
created_at: 2025-04-18T18:12:49Z
author_association: MEMBER
diff_hunk: @@ -24,6 +24,12 @@ const (
 	// which is evaluated if the file deletion threshold has been reached. If `captureFileIndex % N == 0`
 	// return true then the file will be deleted to free up space.
 	defaultDeleteEveryNth = 5
+	// defaultDiskUsageThreshold and defaultCaptureDirThresholdPercent are the thresholds at which file deletion might occur.
+	// If disk usage is at or above this threshold, AND the capture directory makes up at least CaptureDirThresholdPercent of the disk usage,
path: services/datamanager/builtin/config.go
user: login: katiepeters
body: ```suggestion
	// If disk usage is at or above this threshold, AND the capture directory makes up at least CaptureDirThreshold (%) of the disk usage,
```
created_at: 2025-04-18T18:13:39Z
author_association: MEMBER
diff_hunk: @@ -137,6 +164,8 @@ func (c *Config) syncConfig(syncSensor sensor.Sensor, syncSensorEnabled bool, lo
 		CaptureDir:                 c.getCaptureDir(logger),
 		CaptureDisabled:            c.CaptureDisabled,
 		DeleteEveryNthWhenDiskFull: c.DeleteEveryNthWhenDiskFull,
+		DiskUsageDeletionThreshold: c.DiskUsageDeletionThreshold,
+		CaptureDirToFSThreshold:    c.CaptureDirDeletionThreshold,
path: services/datamanager/builtin/config.go
user: login: katiepeters
body: I think it's confusing
created_at: 2025-04-18T18:16:12Z
author_association: MEMBER

PR comments:
user: login: katiepeters
created_at: 2025-04-17T14:37:44Z
body: @vijayvuyyuru looks like you need to pull in changes from main
author_association: MEMBER
user: login: katiepeters
created_at: 2025-04-17T14:38:12Z
body: Can you include any testing that you did in the description?
author_association: MEMBER