Repository: rclone/rclone
Pull request #4006 (2020-02-25 17:32:36)

Cutoff mode
#### What is the purpose of this change?

Updating PR #3690 for requested changes - Fixing #2672 

I rebased the PR against master, fixing any conflicts, then made all 
 requested changes from the original PR

Original:

Implement max transfer cut off mode.
Added new flags `--max-transfer-(hard,soft,cautious)`

Defaults to `--max-transfer-hard`.

Specifiying `--max-transfer-hard` will stop transferring immediately
when Rclone reaches the limit.

Specifiying `--max-transfer-soft` will stop starting new transfers
when Rclone reaches the limit.
 
Specifiying `--max-transfer-cautious` will try to prevent Rclone
from reaching the limit.
 
#### Was the change discussed in an issue or in the forum before?
The feature is requested in #2672
 
#### Checklist
* [x]  I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
* [x]  I have added tests for all changes in this PR if appropriate.
* [x]  I have added documentation for the changes if appropriate.
* [x]  All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
* [x]  I'm done, this Pull Request is ready for review :-)



ncw: Thank you for this :-)

I've squashed it into 2 commits, one for each author.

I'll merge it now - thank you very much!

Patch:
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 70622bc3f..4a1c341ab 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -736,6 +736,20 @@ When the limit is reached all transfers will stop immediately.
 
 Rclone will exit with exit code 8 if the transfer limit is reached.
 
+### --cutoff-mode=hard|soft|cautious ###
+
+This modifies the behavior of `--max-transfer`
+Defaults to `--cutoff-mode=hard`.
+
+Specifiying `--cutoff-mode=hard` will stop transferring immediately
+when Rclone reaches the limit.
+
+Specifiying `--cutoff-mode=soft` will stop starting new transfers
+when Rclone reaches the limit.
+
+Specifiying `--cutoff-mode=cautious` will try to prevent Rclone
+from reaching the limit.
+
 ### --modify-window=TIME ###
 
 When checking whether a file has been modified, this is the maximum
diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go
index dd9a5145f..eb280b0a9 100644
--- a/fs/accounting/accounting.go
+++ b/fs/accounting/accounting.go
@@ -61,7 +61,10 @@ func newAccountSizeName(stats *StatsInfo, in io.ReadCloser, size int64, name str
 		exit:   make(chan struct{}),
 		avg:    0,
 		lpTime: time.Now(),
-		max:    int64(fs.Config.MaxTransfer),
+		max:    -1,
+	}
+	if fs.Config.CutoffMode == fs.CutoffModeHard {
+		acc.max = int64((fs.Config.MaxTransfer))
 	}
 	go acc.averageLoop()
 	stats.inProgress.set(acc.name, acc)
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index 993afdcba..a61b88886 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -382,6 +382,22 @@ func (s *StatsInfo) GetBytes() int64 {
 	return s.bytes
 }
 
+// GetBytesWithPending returns the number of bytes transferred and remaining transfers
+func (s *StatsInfo) GetBytesWithPending() int64 {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	pending := int64(0)
+	for _, tr := range s.startedTransfers {
+		if tr.acc != nil {
+			bytes, size := tr.acc.progress()
+			if bytes < size {
+				pending += size - bytes
+			}
+		}
+	}
+	return s.bytes + pending
+}
+
 // Errors updates the stats for errors
 func (s *StatsInfo) Errors(errors int64) {
 	s.mu.Lock()
diff --git a/fs/config.go b/fs/config.go
index 661cf405a..1bc6457bf 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -93,6 +93,7 @@ type ConfigInfo struct {
 	UseServerModTime       bool
 	MaxTransfer            SizeSuffix
 	MaxDuration            time.Duration
+	CutoffMode             CutoffMode
 	MaxBacklog             int
 	MaxStatsGroups         int
 	StatsOneLine           bool
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index a70777a0a..e87d0d8ae 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -94,6 +94,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList)
 	flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.")
 	flags.DurationVarP(flagSet, &fs.Config.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.")
+	flags.FVarP(flagSet, &fs.Config.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS")
 	flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.")
 	flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.")
 	flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.")
diff --git a/fs/cutoffmode.go b/fs/cutoffmode.go
new file mode 100644
index 000000000..19ec2b0c4
--- /dev/null
+++ b/fs/cutoffmode.go
@@ -0,0 +1,49 @@
+package fs
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// CutoffMode describes the possible delete modes in the config
+type CutoffMode byte
+
+// MaxTransferMode constants
+const (
+	CutoffModeHard CutoffMode = iota
+	CutoffModeSoft
+	CutoffModeCautious
+	CutoffModeDefault = CutoffModeHard
+)
+
+var cutoffModeToString = []string{
+	CutoffModeHard:     "HARD",
+	CutoffModeSoft:     "SOFT",
+	CutoffModeCautious: "CAUTIOUS",
+}
+
+// String turns a LogLevel into a string
+func (m CutoffMode) String() string {
+	if m >= CutoffMode(len(cutoffModeToString)) {
+		return fmt.Sprintf("CutoffMode(%d)", m)
+	}
+	return cutoffModeToString[m]
+}
+
+// Set a LogLevel
+func (m *CutoffMode) Set(s string) error {
+	for n, name := range cutoffModeToString {
+		if s != "" && name == strings.ToUpper(s) {
+			*m = CutoffMode(n)
+			return nil
+		}
+	}
+	return errors.Errorf("Unknown cutoff mode %q", s)
+}
+
+// Type of the value
+func (m *CutoffMode) Type() string {
+	return "string"
+}
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index c04b5ab77..ce4f24a5c 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -362,11 +362,11 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 		// Try server side copy first - if has optional interface and
 		// is same underlying remote
 		actionTaken = "Copied (server side copy)"
+		if fs.Config.MaxTransfer >= 0 && (accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) ||
+			(fs.Config.CutoffMode == fs.CutoffModeCautious && accounting.Stats(ctx).GetBytesWithPending()+src.Size() >= int64(fs.Config.MaxTransfer))) {
+			return nil, accounting.ErrorMaxTransferLimitReached
+		}
 		if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) {
-			// Check transfer limit for server side copies
-			if fs.Config.MaxTransfer >= 0 && accounting.Stats(ctx).GetBytes() >= int64(fs.Config.MaxTransfer) {
-				return nil, accounting.ErrorMaxTransferLimitReached
-			}
 			in := tr.Account(nil) // account the transfer
 			in.ServerSideCopyStart()
 			newDst, err = doCopy(ctx, src, remote)


Test patch:
diff --git a/fs/accounting/accounting_test.go b/fs/accounting/accounting_test.go
index 3f365e493..9a00f48dd 100644
--- a/fs/accounting/accounting_test.go
+++ b/fs/accounting/accounting_test.go
@@ -197,9 +197,12 @@ func TestAccountAccounter(t *testing.T) {
 
 func TestAccountMaxTransfer(t *testing.T) {
 	old := fs.Config.MaxTransfer
+	oldMode := fs.Config.CutoffMode
+
 	fs.Config.MaxTransfer = 15
 	defer func() {
 		fs.Config.MaxTransfer = old
+		fs.Config.CutoffMode = oldMode
 	}()
 
 	in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100)))
@@ -218,6 +221,20 @@ func TestAccountMaxTransfer(t *testing.T) {
 	assert.Equal(t, 0, n)
 	assert.Equal(t, ErrorMaxTransferLimitReached, err)
 	assert.True(t, fserrors.IsFatalError(err))
+
+	fs.Config.CutoffMode = fs.CutoffModeSoft
+	stats = NewStats()
+	acc = newAccountSizeName(stats, in, 1, "test")
+
+	n, err = acc.Read(b)
+	assert.Equal(t, 10, n)
+	assert.NoError(t, err)
+	n, err = acc.Read(b)
+	assert.Equal(t, 10, n)
+	assert.NoError(t, err)
+	n, err = acc.Read(b)
+	assert.Equal(t, 10, n)
+	assert.NoError(t, err)
 }
 
 func TestShortenName(t *testing.T) {
diff --git a/fs/cutoffmode_test.go b/fs/cutoffmode_test.go
new file mode 100644
index 000000000..0fb7cc562
--- /dev/null
+++ b/fs/cutoffmode_test.go
@@ -0,0 +1,6 @@
+package fs
+
+import "github.com/spf13/pflag"
+
+// Check it satisfies the interface
+var _ pflag.Value = (*CutoffMode)(nil)
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index ddefac822..192dd6610 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -39,6 +39,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/filter"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
@@ -1527,3 +1528,58 @@ func TestRcatSize(t *testing.T) {
 	// Check files exist
 	fstest.CheckItems(t, r.Fremote, file1, file2)
 }
+
+func TestCopyFileMaxTransfer(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	old := fs.Config.MaxTransfer
+	oldMode := fs.Config.CutoffMode
+
+	defer func() {
+		fs.Config.MaxTransfer = old
+		fs.Config.CutoffMode = oldMode
+		accounting.Stats(context.Background()).ResetCounters()
+	}()
+
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	file2 := r.WriteFile("file2", "file2 contents...........", t2)
+
+	rfile1 := file1
+	rfile1.Path = "sub/file1"
+	rfile2 := file2
+	rfile2.Path = "sub/file2"
+
+	fs.Config.MaxTransfer = 15
+	fs.Config.CutoffMode = fs.CutoffModeHard
+	accounting.Stats(context.Background()).ResetCounters()
+
+	err := operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile1.Path, file1.Path)
+	require.NoError(t, err)
+	fstest.CheckItems(t, r.Flocal, file1, file2)
+	fstest.CheckItems(t, r.Fremote, rfile1)
+
+	accounting.Stats(context.Background()).ResetCounters()
+
+	err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
+	fstest.CheckItems(t, r.Flocal, file1, file2)
+	fstest.CheckItems(t, r.Fremote, rfile1)
+	assert.Contains(t, err.Error(), "Max transfer limit reached")
+	assert.True(t, fserrors.IsFatalError(err))
+
+	fs.Config.CutoffMode = fs.CutoffModeCautious
+	accounting.Stats(context.Background()).ResetCounters()
+
+	err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
+	fstest.CheckItems(t, r.Flocal, file1, file2)
+	fstest.CheckItems(t, r.Fremote, rfile1)
+	assert.Contains(t, err.Error(), "Max transfer limit reached")
+	assert.True(t, fserrors.IsFatalError(err))
+
+	fs.Config.CutoffMode = fs.CutoffModeSoft
+	accounting.Stats(context.Background()).ResetCounters()
+
+	err = operations.CopyFile(context.Background(), r.Fremote, r.Flocal, rfile2.Path, file2.Path)
+	require.NoError(t, err)
+	fstest.CheckItems(t, r.Flocal, file1, file2)
+	fstest.CheckItems(t, r.Fremote, rfile1, rfile2)
+}