Repository: rclone/rclone (language: go)

Issue:
Backblaze B2 remote does not always honor `--dry-run` or `--interactive` flags
#### What is the problem you are having with rclone?

The Backblaze B2 remote does not honor the `--dry-run` or `--interactive` flags in `backend` operations.

#### What is your rclone version (output from `rclone version`)

```
% rclone version
rclone v1.68.2
- os/version: darwin 14.7.1 (64 bit)
- os/kernel: 23.6.0 (arm64)
- os/type: darwin
- os/arch: arm64 (ARMv8 compatible)
- go/version: go1.23.3
- go/linking: dynamic
- go/tags: cmount
```

#### Which OS you are using and how many bits (e.g. Windows 7, 64 bit)

macOS Sonoma 14.7.1

#### Which cloud storage system are you using? (e.g. Google Drive)

Backblaze B2

#### The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)

`rclone backend cleanup-hidden b2:metadaddy-rclone --dry-run`

#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)

First, create some hidden versions...

```
% rclone copyto -I /Users/ppatterson/Documents/Test\ Files/hello.txt b2:metadaddy-rclone/test-dry-run.txt
% rclone copyto -I /Users/ppatterson/Documents/Test\ Files/hello.txt b2:metadaddy-rclone/test-dry-run.txt
% rclone copyto -I /Users/ppatterson/Documents/Test\ Files/hello.txt b2:metadaddy-rclone/test-dry-run.txt
% rclone --b2-versions ls b2:metadaddy-rclone       
       14 test-dry-run.txt
       14 test-dry-run-v2024-11-19-014429-131.txt
       14 test-dry-run-v2024-11-19-014219-013.txt
```

Now run `cleanup-hidden` with `--dry-run`:

```
% rclone backend cleanup-hidden b2:metadaddy-rclone --dry-run -vv                                  
2024/11/18 17:42:32 DEBUG : Setting --fast-list "true" from environment variable RCLONE_FAST_LIST="true"
2024/11/18 17:42:32 DEBUG : rclone: Version "v1.68.2" starting with parameters ["rclone" "backend" "cleanup-hidden" "b2:metadaddy-rclone" "--dry-run" "-vv"]
2024/11/18 17:42:32 DEBUG : Using config file from "/Users/ppatterson/.config/rclone/rclone.conf"
2024/11/18 17:42:32 INFO  : B2 bucket metadaddy-rclone: cleaning bucket "metadaddy-rclone" of all hidden files
2024/11/18 17:42:32 DEBUG : test-dry-run.txt: Not deleting current version (id "4_z91a5cf5973f5dc4f848d0c1b_f10733d4f99d18f52_d20241119_m014219_c004_v0402027_t0044_u01731980539013") "upload" dated 2024-11-18 17:42:19.013 -0800 PST (13.651871s ago)
2024/11/18 17:42:32 DEBUG : test-dry-run.txt: Deleting (id "4_z91a5cf5973f5dc4f848d0c1b_f118ed7238ae3ae95_d20241119_m014215_c004_v0402026_t0034_u01731980535576")
2024/11/18 17:42:32 DEBUG : test-dry-run.txt: Deleting (id "4_z91a5cf5973f5dc4f848d0c1b_f1175cd5b88012b9d_d20241119_m014212_c004_v0402007_t0007_u01731980532933")
2024/11/18 17:42:32 DEBUG : 11 go routines active
```

Old versions are gone!

```
% rclone --b2-versions ls b2:metadaddy-rclone                    
       14 test-dry-run.txt
```

Looking at the code, `operations.SkipDestructive()` is not called at all in `b2.go`.

I'll see if I can spin up a quick PR - it looks like it's just a case of adding `operations.SkipDestructive()` and the associated logic in the relevant places.

<!--- Please keep the note below for others who read your bug report. -->

#### How to use GitHub

* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
* Subscribe to receive notifications on status change and new comments.



Discussion / hints:

Fix patch:
diff --git a/backend/b2/b2.go b/backend/b2/b2.go
index 4d4e754242566..ad58383f9ac49 100644
--- a/backend/b2/b2.go
+++ b/backend/b2/b2.go
@@ -30,6 +30,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
@@ -1318,16 +1319,22 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
 				// Check current version of the file
 				if deleteHidden && object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
-					toBeDeleted <- object
+					if !operations.SkipDestructive(ctx, object.Name, "remove hide marker") {
+						toBeDeleted <- object
+					}
 				} else if deleteUnfinished && object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
-					toBeDeleted <- object
+					if !operations.SkipDestructive(ctx, object.Name, "remove pending upload") {
+						toBeDeleted <- object
+					}
 				} else {
 					fs.Debugf(remote, "Not deleting current version (id %q) %q dated %v (%v ago)", object.ID, object.Action, time.Time(object.UploadTimestamp).Local(), time.Since(time.Time(object.UploadTimestamp)))
 				}
 			} else {
 				fs.Debugf(remote, "Deleting (id %q)", object.ID)
-				toBeDeleted <- object
+				if !operations.SkipDestructive(ctx, object.Name, "delete") {
+					toBeDeleted <- object
+				}
 			}
 			last = remote
 			tr.Done(ctx, nil)
@@ -2293,8 +2300,10 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op
 
 	}
 
+	skip := operations.SkipDestructive(ctx, name, "update lifecycle rules")
+
 	var bucket *api.Bucket
-	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil {
+	if !skip && (newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil) {
 		bucketID, err := f.getBucketID(ctx, bucketName)
 		if err != nil {
 			return nil, err


Test patch:
diff --git a/backend/b2/b2_internal_test.go b/backend/b2/b2_internal_test.go
index e6bd8b404db1d..c470dcc4d8697 100644
--- a/backend/b2/b2_internal_test.go
+++ b/backend/b2/b2_internal_test.go
@@ -5,6 +5,7 @@ import (
 	"crypto/sha1"
 	"fmt"
 	"path"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -13,6 +14,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/bucket"
@@ -463,24 +465,161 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 	})
 
 	t.Run("Cleanup", func(t *testing.T) {
-		require.NoError(t, f.cleanUp(ctx, true, false, 0))
-		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
-		fstest.CheckListing(t, f, items)
-		// Set --b2-versions for this test
-		f.opt.Versions = true
-		defer func() {
-			f.opt.Versions = false
-		}()
-		fstest.CheckListing(t, f, items)
+		t.Run("DryRun", func(t *testing.T) {
+			f.opt.Versions = true
+			defer func() {
+				f.opt.Versions = false
+			}()
+			// Listing should be unchanged after dry run
+			before := listAllFiles(ctx, t, f, dirName)
+			ctx, ci := fs.AddConfig(ctx)
+			ci.DryRun = true
+			require.NoError(t, f.cleanUp(ctx, true, false, 0))
+			after := listAllFiles(ctx, t, f, dirName)
+			assert.Equal(t, before, after)
+		})
+
+		t.Run("RealThing", func(t *testing.T) {
+			f.opt.Versions = true
+			defer func() {
+				f.opt.Versions = false
+			}()
+			// Listing should reflect current state after cleanup
+			require.NoError(t, f.cleanUp(ctx, true, false, 0))
+			items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
+			fstest.CheckListing(t, f, items)
+		})
 	})
 
 	// Purge gets tested later
 }
 
+func (f *Fs) InternalTestCleanupUnfinished(t *testing.T) {
+	ctx := context.Background()
+
+	// B2CleanupHidden tests cleaning up hidden files
+	t.Run("CleanupUnfinished", func(t *testing.T) {
+		dirName := "unfinished"
+		fileCount := 5
+		expectedFiles := []string{}
+		for i := 1; i < fileCount; i++ {
+			fileName := fmt.Sprintf("%s/unfinished-%d", dirName, i)
+			expectedFiles = append(expectedFiles, fileName)
+			obj := &Object{
+				fs:     f,
+				remote: fileName,
+			}
+			objInfo := object.NewStaticObjectInfo(fileName, fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil)
+			_, err := f.newLargeUpload(ctx, obj, nil, objInfo, f.opt.ChunkSize, false, nil)
+			require.NoError(t, err)
+		}
+		checkListing(ctx, t, f, dirName, expectedFiles)
+
+		t.Run("DryRun", func(t *testing.T) {
+			// Listing should not change after dry run
+			ctx, ci := fs.AddConfig(ctx)
+			ci.DryRun = true
+			require.NoError(t, f.cleanUp(ctx, false, true, 0))
+			checkListing(ctx, t, f, dirName, expectedFiles)
+		})
+
+		t.Run("RealThing", func(t *testing.T) {
+			// Listing should be empty after real cleanup
+			require.NoError(t, f.cleanUp(ctx, false, true, 0))
+			checkListing(ctx, t, f, dirName, []string{})
+		})
+	})
+}
+
+func listAllFiles(ctx context.Context, t *testing.T, f *Fs, dirName string) []string {
+	bucket, directory := f.split(dirName)
+	foundFiles := []string{}
+	require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
+		if !isDirectory {
+			foundFiles = append(foundFiles, object.Name)
+		}
+		return nil
+	}))
+	sort.Strings(foundFiles)
+	return foundFiles
+}
+
+func checkListing(ctx context.Context, t *testing.T, f *Fs, dirName string, expectedFiles []string) {
+	foundFiles := listAllFiles(ctx, t, f, dirName)
+	sort.Strings(expectedFiles)
+	assert.Equal(t, expectedFiles, foundFiles)
+}
+
+func (f *Fs) InternalTestLifecycleRules(t *testing.T) {
+	ctx := context.Background()
+
+	opt := map[string]string{}
+
+	t.Run("InitState", func(t *testing.T) {
+		// There should be no lifecycle rules at the outset
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+	})
+
+	t.Run("DryRun", func(t *testing.T) {
+		// There should still be no lifecycle rules after each dry run operation
+		ctx, ci := fs.AddConfig(ctx)
+		ci.DryRun = true
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+
+		delete(opt, "daysFromHidingToDeleting")
+		opt["daysFromUploadingToHiding"] = "40"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(lifecycleRules))
+	})
+
+	t.Run("RealThing", func(t *testing.T) {
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)
+
+		delete(opt, "daysFromHidingToDeleting")
+		opt["daysFromUploadingToHiding"] = "40"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)
+
+		opt["daysFromHidingToDeleting"] = "30"
+		lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)
+		lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(lifecycleRules))
+		assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)
+		assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)
+	})
+}
+
 // -run TestIntegration/FsMkdir/FsPutFiles/Internal
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Metadata", f.InternalTestMetadata)
 	t.Run("Versions", f.InternalTestVersions)
+	t.Run("CleanupUnfinished", f.InternalTestCleanupUnfinished)
+	t.Run("LifecycleRules", f.InternalTestLifecycleRules)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)