Repository: rclone/rclone
Pull request #7608 (2024-02-01 16:28:24)

quatrix: fix Content-Range header
This change does not actually affect uploads. Just to be right according to definition of Content-Range in https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range#range-end

<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?

Content-Range is inclusive according to MDN

#### Was the change discussed in an issue or in the forum before?

No

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)


vkit-maytech: Hi @ncw
Please also check this PR  https://github.com/rclone/rclone/pull/7525 that fixes `TestIntegration/FsMkdir/FsPutFiles/FsIsFile/FsRoot` test for Quatrix

Patch:
diff --git a/backend/b2/b2.go b/backend/b2/b2.go
index 778d856f1..50413cc03 100644
--- a/backend/b2/b2.go
+++ b/backend/b2/b2.go
@@ -193,9 +193,12 @@ Example:
 			Advanced: true,
 		}, {
 			Name: "download_auth_duration",
-			Help: `Time before the authorization token will expire in s or suffix ms|s|m|h|d.
+			Help: `Time before the public link authorization token will expire in s or suffix ms|s|m|h|d.
+
+This is used in combination with "rclone link" for making files
+accessible to the public and sets the duration before the download
+authorization token will expire.
 
-The duration before the download authorization token will expire.
 The minimum value is 1 second. The maximum value is one week.`,
 			Default:  fs.Duration(7 * 24 * time.Hour),
 			Advanced: true,
diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go
index aab419779..6c7e7b105 100644
--- a/backend/ftp/ftp.go
+++ b/backend/ftp/ftp.go
@@ -970,6 +970,8 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	f.putFtpConnection(&c, err)
 	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
+		case ftp.StatusRequestedFileActionOK: // some ftp servers apparently return 250 instead of 257
+			err = nil // see: https://forum.rclone.org/t/rclone-pop-up-an-i-o-error-when-creating-a-folder-in-a-mounted-ftp-drive/44368/
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
 		case 521: // dir already exists: error number according to RFC 959: issue #2363
diff --git a/backend/quatrix/quatrix.go b/backend/quatrix/quatrix.go
index dab1f5f77..e615ac49e 100644
--- a/backend/quatrix/quatrix.go
+++ b/backend/quatrix/quatrix.go
@@ -1198,11 +1198,12 @@ func (o *Object) uploadSession(ctx context.Context, parentID, name string) (uplo
 
 func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.Reader, fullSize int64, offset int64, chunkSize int64, options ...fs.OpenOption) (err error) {
 	opts := rest.Opts{
-		Method:       "POST",
-		RootURL:      fmt.Sprintf(uploadURL, o.fs.opt.Host) + uploadKey,
-		Body:         chunk,
-		ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize, fullSize),
-		Options:      options,
+		Method:        "POST",
+		RootURL:       fmt.Sprintf(uploadURL, o.fs.opt.Host) + uploadKey,
+		Body:          chunk,
+		ContentLength: &chunkSize,
+		ContentRange:  fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, fullSize),
+		Options:       options,
 	}
 
 	var fileID string
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 473f74ece..9183670c4 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3222,6 +3222,10 @@ func setQuirks(opt *Options) {
 		// https://github.com/rclone/rclone/issues/6670
 		useAcceptEncodingGzip = false
 		useAlreadyExists = true // returns BucketNameUnavailable instead of BucketAlreadyExists but good enough!
+		// GCS S3 doesn't support multi-part server side copy:
+		// See: https://issuetracker.google.com/issues/323465186
+		// So make cutoff very large which it does seem to support
+		opt.CopyCutoff = math.MaxInt64
 	default:
 		fs.Logf("s3", "s3 provider %q not known - please set correctly", opt.Provider)
 		fallthrough
diff --git a/backend/seafile/webapi.go b/backend/seafile/webapi.go
index d3ebe3fde..1cc05a14b 100644
--- a/backend/seafile/webapi.go
+++ b/backend/seafile/webapi.go
@@ -577,7 +577,7 @@ func (f *Fs) getDownloadLink(ctx context.Context, libraryID, filePath string) (s
 	return result, nil
 }
 
-func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs.OpenOption) (io.ReadCloser, error) {
+func (f *Fs) download(ctx context.Context, downloadLink string, size int64, options ...fs.OpenOption) (io.ReadCloser, error) {
 	// Check if we need to download partial content
 	var start, end int64 = 0, size
 	partialContent := false
@@ -606,11 +606,18 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
 	// Build the http request
 	opts := rest.Opts{
 		Method:  "GET",
-		RootURL: url,
 		Options: options,
 	}
+	parsedURL, err := url.Parse(downloadLink)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse download url: %w", err)
+	}
+	if parsedURL.IsAbs() {
+		opts.RootURL = downloadLink
+	} else {
+		opts.Path = downloadLink
+	}
 	var resp *http.Response
-	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
 		return f.shouldRetry(ctx, resp, err)
@@ -618,7 +625,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
 	if err != nil {
 		if resp != nil {
 			if resp.StatusCode == 404 {
-				return nil, fmt.Errorf("file not found '%s'", url)
+				return nil, fmt.Errorf("file not found '%s'", downloadLink)
 			}
 		}
 		return nil, err
@@ -688,11 +695,19 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
 
 	opts := rest.Opts{
 		Method:      "POST",
-		RootURL:     uploadLink,
 		Body:        formReader,
 		ContentType: contentType,
 		Parameters:  url.Values{"ret-json": {"1"}}, // It needs to be on the url, not in the body parameters
 	}
+	parsedURL, err := url.Parse(uploadLink)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse upload url: %w", err)
+	}
+	if parsedURL.IsAbs() {
+		opts.RootURL = uploadLink
+	} else {
+		opts.Path = uploadLink
+	}
 	result := make([]api.FileDetail, 1)
 	var resp *http.Response
 	// If an error occurs during the call, do not attempt to retry: The upload link is single use only
diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go
index 3c24c3583..2750e6458 100644
--- a/cmd/serve/ftp/ftp.go
+++ b/cmd/serve/ftp/ftp.go
@@ -339,7 +339,7 @@ func (d *driver) ListDir(sctx *ftp.Context, path string, callback func(iofs.File
 	}
 
 	// Account the transfer
-	tr := accounting.GlobalStats().NewTransferRemoteSize(path, node.Size())
+	tr := accounting.GlobalStats().NewTransferRemoteSize(path, node.Size(), d.f, nil)
 	defer func() {
 		tr.Done(d.ctx, err)
 	}()
@@ -448,7 +448,7 @@ func (d *driver) GetFile(sctx *ftp.Context, path string, offset int64) (size int
 	}
 
 	// Account the transfer
-	tr := accounting.GlobalStats().NewTransferRemoteSize(path, node.Size())
+	tr := accounting.GlobalStats().NewTransferRemoteSize(path, node.Size(), d.f, nil)
 	defer tr.Done(d.ctx, nil)
 
 	return node.Size(), handle, nil
diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index 6af35ccd9..e3e373acc 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -297,7 +297,7 @@ func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string)
 	}()
 
 	// Account the transfer
-	tr := accounting.Stats(r.Context()).NewTransfer(obj)
+	tr := accounting.Stats(r.Context()).NewTransfer(obj, nil)
 	defer tr.Done(r.Context(), nil)
 	// FIXME in = fs.NewAccount(in, obj).WithBuffer() // account the transfer
 
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 9b588ae11..cd692ed5a 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -817,3 +817,6 @@ put them back in again.` >}}
   * Harshit Budhraja <52413945+harshit-budhraja@users.noreply.github.com>
   * Tera <24725862+teraa@users.noreply.github.com>
   * Kyle Reynolds <kylereynoldsdev@gmail.com>
+  * Michael Eischer <michael.eischer@gmx.de>
+  * Thomas Müller <1005065+DeepDiver1975@users.noreply.github.com>
+  * DanielEgbers <27849724+DanielEgbers@users.noreply.github.com>
diff --git a/docs/content/rc.md b/docs/content/rc.md
index 7e4ed11d0..9fb42917a 100644
--- a/docs/content/rc.md
+++ b/docs/content/rc.md
@@ -847,12 +847,15 @@ Returns the following values:
 		[
 			{
 				"bytes": total transferred bytes for this file,
-				"eta": estimated time in seconds until file transfer completion
+				"eta": estimated time in seconds until file transfer completion (may be nil)
 				"name": name of the file,
 				"percentage": progress of the file transfer in percent,
 				"speed": average speed over the whole transfer in bytes per second,
 				"speedAvg": current speed in bytes per second as an exponentially weighted moving average,
 				"size": size of the file in bytes
+				"group": stats group this transfer is part of
+				"srcFs": name of the source remote (not present if not known)
+				"dstFs": name of the destination remote (not present if not known)
 			}
 		],
 	"checking": an array of names of currently active file checks
@@ -904,9 +907,12 @@ Returns the following values:
 				"size": size of the file in bytes,
 				"bytes": total transferred bytes for this file,
 				"checked": if the transfer is only checked (skipped, deleted),
-				"timestamp": integer representing millisecond unix epoch,
+				"started_at": time the transfer was started at (RFC3339 format, eg `"2000-01-01T01:00:00.085742121Z"`),
+				"completed_at": time the transfer was completed at (RFC3339 format, only present if transfer is completed),
 				"error": string description of the error (empty if successful),
-				"jobid": id of the job that this transfer belongs to
+				"group": string representing which stats group this is part of,
+				"srcFs": name of the source remote (not present if not known),
+				"dstFs": name of the destination remote (not present if not known),
 			}
 		]
 }
diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go
index 41eb71225..2aa8c3219 100644
--- a/fs/accounting/accounting.go
+++ b/fs/accounting/accounting.go
@@ -539,9 +539,8 @@ func (acc *Account) String() string {
 	)
 }
 
-// rcStats produces remote control stats for this file
-func (acc *Account) rcStats() (out rc.Params) {
-	out = make(rc.Params)
+// rcStats adds remote control stats for this file
+func (acc *Account) rcStats(out rc.Params) {
 	a, b := acc.progress()
 	out["bytes"] = a
 	out["size"] = b
@@ -563,8 +562,6 @@ func (acc *Account) rcStats() (out rc.Params) {
 	}
 	out["percentage"] = percentageDone
 	out["group"] = acc.stats.group
-
-	return out
 }
 
 // OldStream returns the top io.Reader
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index 3d89defd4..393b6711a 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -775,16 +775,24 @@ func (s *StatsInfo) GetTransfers() int64 {
 }
 
 // NewTransfer adds a transfer to the stats from the object.
-func (s *StatsInfo) NewTransfer(obj fs.DirEntry) *Transfer {
-	tr := newTransfer(s, obj)
+//
+// The obj is uses as the srcFs, the dstFs must be supplied
+func (s *StatsInfo) NewTransfer(obj fs.DirEntry, dstFs fs.Fs) *Transfer {
+	var srcFs fs.Fs
+	if oi, ok := obj.(fs.ObjectInfo); ok {
+		if f, ok := oi.Fs().(fs.Fs); ok {
+			srcFs = f
+		}
+	}
+	tr := newTransfer(s, obj, srcFs, dstFs)
 	s.transferring.add(tr)
 	s.startAverageLoop()
 	return tr
 }
 
 // NewTransferRemoteSize adds a transfer to the stats based on remote and size.
-func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64) *Transfer {
-	tr := newTransferRemoteSize(s, remote, size, false, "")
+func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64, srcFs, dstFs fs.Fs) *Transfer {
+	tr := newTransferRemoteSize(s, remote, size, false, "", srcFs, dstFs)
 	s.transferring.add(tr)
 	s.startAverageLoop()
 	return tr
diff --git a/fs/accounting/transfer.go b/fs/accounting/transfer.go
index 5860eefb2..f2bc67761 100644
--- a/fs/accounting/transfer.go
+++ b/fs/accounting/transfer.go
@@ -21,6 +21,8 @@ type TransferSnapshot struct {
 	CompletedAt time.Time `json:"completed_at,omitempty"`
 	Error       error     `json:"-"`
 	Group       string    `json:"group"`
+	SrcFs       string    `json:"srcFs,omitempty"`
+	DstFs       string    `json:"dstFs,omitempty"`
 }
 
 // MarshalJSON implements json.Marshaler interface.
@@ -51,6 +53,8 @@ type Transfer struct {
 	startedAt time.Time
 	checking  bool
 	what      string // what kind of transfer this is
+	srcFs     fs.Fs  // source Fs - may be nil
+	dstFs     fs.Fs  // destination Fs - may be nil
 
 	// Protects all below
 	//
@@ -65,15 +69,15 @@ type Transfer struct {
 
 // newCheckingTransfer instantiates new checking of the object.
 func newCheckingTransfer(stats *StatsInfo, obj fs.DirEntry, what string) *Transfer {
-	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true, what)
+	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true, what, nil, nil)
 }
 
 // newTransfer instantiates new transfer.
-func newTransfer(stats *StatsInfo, obj fs.DirEntry) *Transfer {
-	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false, "")
+func newTransfer(stats *StatsInfo, obj fs.DirEntry, srcFs, dstFs fs.Fs) *Transfer {
+	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false, "", srcFs, dstFs)
 }
 
-func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking bool, what string) *Transfer {
+func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking bool, what string, srcFs, dstFs fs.Fs) *Transfer {
 	tr := &Transfer{
 		stats:     stats,
 		remote:    remote,
@@ -81,6 +85,8 @@ func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking
 		startedAt: time.Now(),
 		checking:  checking,
 		what:      what,
+		srcFs:     srcFs,
+		dstFs:     dstFs,
 	}
 	stats.AddTransfer(tr)
 	return tr
@@ -178,7 +184,7 @@ func (tr *Transfer) Snapshot() TransferSnapshot {
 	if tr.acc != nil {
 		b, s = tr.acc.progress()
 	}
-	return TransferSnapshot{
+	snapshot := TransferSnapshot{
 		Name:        tr.remote,
 		Checked:     tr.checking,
 		Size:        s,
@@ -188,12 +194,26 @@ func (tr *Transfer) Snapshot() TransferSnapshot {
 		Error:       tr.err,
 		Group:       tr.stats.group,
 	}
+	if tr.srcFs != nil {
+		snapshot.SrcFs = fs.ConfigString(tr.srcFs)
+	}
+	if tr.dstFs != nil {
+		snapshot.DstFs = fs.ConfigString(tr.dstFs)
+	}
+	return snapshot
 }
 
 // rcStats returns stats for the transfer suitable for the rc
 func (tr *Transfer) rcStats() rc.Params {
-	return rc.Params{
+	out := rc.Params{
 		"name": tr.remote, // no locking needed to access this
 		"size": tr.size,
 	}
+	if tr.srcFs != nil {
+		out["srcFs"] = fs.ConfigString(tr.srcFs)
+	}
+	if tr.dstFs != nil {
+		out["dstFs"] = fs.ConfigString(tr.dstFs)
+	}
+	return out
 }
diff --git a/fs/accounting/transfermap.go b/fs/accounting/transfermap.go
index 518124ffc..41409eece 100644
--- a/fs/accounting/transfermap.go
+++ b/fs/accounting/transfermap.go
@@ -159,11 +159,11 @@ func (tm *transferMap) rcStats(progress *inProgress) (t []rc.Params) {
 	tm.mu.RLock()
 	defer tm.mu.RUnlock()
 	for _, tr := range tm._sortedSlice() {
+		out := tr.rcStats() // basic stats
 		if acc := progress.get(tr.remote); acc != nil {
-			t = append(t, acc.rcStats())
-		} else {
-			t = append(t, tr.rcStats())
+			acc.rcStats(out) // add extended stats if have acc
 		}
+		t = append(t, out)
 	}
 	return t
 }
diff --git a/fs/operations/check.go b/fs/operations/check.go
index 328be2128..2279b3fe6 100644
--- a/fs/operations/check.go
+++ b/fs/operations/check.go
@@ -341,7 +341,7 @@ func checkIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ boo
 	if err != nil {
 		return true, fmt.Errorf("failed to open %q: %w", dst, err)
 	}
-	tr1 := accounting.Stats(ctx).NewTransfer(dst)
+	tr1 := accounting.Stats(ctx).NewTransfer(dst, nil)
 	defer func() {
 		tr1.Done(ctx, nil) // error handling is done by the caller
 	}()
@@ -351,7 +351,7 @@ func checkIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ boo
 	if err != nil {
 		return true, fmt.Errorf("failed to open %q: %w", src, err)
 	}
-	tr2 := accounting.Stats(ctx).NewTransfer(dst)
+	tr2 := accounting.Stats(ctx).NewTransfer(dst, nil)
 	defer func() {
 		tr2.Done(ctx, nil) // error handling is done by the caller
 	}()
@@ -501,7 +501,7 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,
 		if in, err = Open(ctx, obj); err != nil {
 			return
 		}
-		tr := accounting.Stats(ctx).NewTransfer(obj)
+		tr := accounting.Stats(ctx).NewTransfer(obj, nil)
 		in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer
 		defer func() {
 			tr.Done(ctx, nil) // will close the stream
diff --git a/fs/operations/copy.go b/fs/operations/copy.go
index 596c7ed04..4425c4974 100644
--- a/fs/operations/copy.go
+++ b/fs/operations/copy.go
@@ -369,7 +369,7 @@ func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) {
 // be nil.
 func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
 	ci := fs.GetConfig(ctx)
-	tr := accounting.Stats(ctx).NewTransfer(src)
+	tr := accounting.Stats(ctx).NewTransfer(src, f)
 	defer func() {
 		tr.Done(ctx, err)
 	}()
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 78b271765..c68f468ff 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -382,7 +382,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
 	ci := fs.GetConfig(ctx)
 	var tr *accounting.Transfer
 	if isTransfer {
-		tr = accounting.Stats(ctx).NewTransfer(src)
+		tr = accounting.Stats(ctx).NewTransfer(src, fdst)
 	} else {
 		tr = accounting.Stats(ctx).NewCheckingTransfer(src, "moving")
 	}
@@ -401,10 +401,16 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
 	// See if we have Move available
 	if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
 		// Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)
-		if dst != nil && !SameObject(src, dst) {
-			err = DeleteFile(ctx, dst)
-			if err != nil {
-				return newDst, err
+		if dst != nil {
+			if !SameObject(src, dst) {
+				err = DeleteFile(ctx, dst)
+				if err != nil {
+					return newDst, err
+				}
+			} else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) {
+				doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+					return moveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src)
+				}
 			}
 		}
 		// Move dst <- src
@@ -814,7 +820,7 @@ func HashSum(ctx context.Context, ht hash.Type, base64Encoded bool, downloadFlag
 		// Setup: Define accounting, open the file with NewReOpen to provide restarts, account for the transfer, and setup a multi-hasher with the appropriate type
 		// Execution: io.Copy file to hasher, get hash and encode in hex
 
-		tr := accounting.Stats(ctx).NewTransfer(o)
+		tr := accounting.Stats(ctx).NewTransfer(o, nil)
 		defer func() {
 			tr.Done(ctx, err)
 		}()
@@ -1106,7 +1112,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []b
 	ci := fs.GetConfig(ctx)
 	return ListFn(ctx, f, func(o fs.Object) {
 		var err error
-		tr := accounting.Stats(ctx).NewTransfer(o)
+		tr := accounting.Stats(ctx).NewTransfer(o, nil)
 		defer func() {
 			tr.Done(ctx, err)
 		}()
@@ -1157,7 +1163,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []b
 // Rcat reads data from the Reader until EOF and uploads it to a file on remote
 func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) {
 	ci := fs.GetConfig(ctx)
-	tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1)
+	tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1, nil, fdst)
 	defer func() {
 		tr.Done(ctx, err)
 	}()
@@ -1603,7 +1609,7 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
 	if size >= 0 {
 		var err error
 		// Size known use Put
-		tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, size)
+		tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, size, nil, fdst)
 		defer func() {
 			tr.Done(ctx, err)
 		}()
@@ -1746,6 +1752,58 @@ func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err err
 	return err
 }
 
+// needsMoveCaseInsensitive returns true if moveCaseInsensitive is needed
+func needsMoveCaseInsensitive(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) bool {
+	dstFilePath := path.Join(fdst.Root(), dstFileName)
+	srcFilePath := path.Join(fsrc.Root(), srcFileName)
+	return !cp && fdst.Name() == fsrc.Name() && fdst.Features().CaseInsensitive && dstFileName != srcFileName && strings.EqualFold(dstFilePath, srcFilePath)
+}
+
+// Special case for changing case of a file on a case insensitive remote
+// This will move the file to a temporary name then
+// move it back to the intended destination. This is required
+// to avoid issues with certain remotes and avoid file deletion.
+// returns nil, nil if !needsMoveCaseInsensitive.
+// this does not account a transfer -- the caller should do that if desired.
+func moveCaseInsensitive(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, srcObj fs.Object) (newDst fs.Object, err error) {
+	if !needsMoveCaseInsensitive(fdst, fsrc, dstFileName, srcFileName, cp) {
+		return nil, nil
+	}
+	logger, _ := GetLogger(ctx)
+
+	// Choose operations
+	Op := MoveTransfer
+	if cp {
+		Op = Copy
+	}
+
+	if SkipDestructive(ctx, srcFileName, "rename to "+dstFileName) {
+		// avoid fatalpanic on --dry-run (trying to access non-existent tmpObj)
+		return nil, nil
+	}
+	// Create random name to temporarily move file to
+	tmpObjName := dstFileName + "-rclone-move-" + random.String(8)
+	tmpObjFail, err := fdst.NewObject(ctx, tmpObjName)
+	if err != fs.ErrorObjectNotFound {
+		if err == nil {
+			logger(ctx, TransferError, nil, tmpObjFail, err)
+			return nil, errors.New("found an already existing file with a randomly generated name. Try the operation again")
+		}
+		logger(ctx, TransferError, nil, tmpObjFail, err)
+		return nil, fmt.Errorf("error while attempting to move file to a temporary location: %w", err)
+	}
+	fs.Debugf(srcObj, "moving to %v", tmpObjName)
+	tmpObj, err := Op(ctx, fdst, nil, tmpObjName, srcObj)
+	if err != nil {
+		logger(ctx, TransferError, srcObj, tmpObj, err)
+		return nil, fmt.Errorf("error while moving file to temporary location: %w", err)
+	}
+	fs.Debugf(srcObj, "moving to %v", dstFileName)
+	newDst, err = Op(ctx, fdst, nil, dstFileName, tmpObj)
+	logger(ctx, MissingOnDst, tmpObj, nil, err)
+	return newDst, err
+}
+
 // moveOrCopyFile moves or copies a single file possibly to a new name
 func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {
 	ci := fs.GetConfig(ctx)
@@ -1791,33 +1849,12 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
 	// This will move the file to a temporary name then
 	// move it back to the intended destination. This is required
 	// to avoid issues with certain remotes and avoid file deletion.
-	if !cp && fdst.Name() == fsrc.Name() && fdst.Features().CaseInsensitive && dstFileName != srcFileName && strings.EqualFold(dstFilePath, srcFilePath) {
-		if SkipDestructive(ctx, srcFileName, "rename to "+dstFileName) {
-			// avoid fatalpanic on --dry-run (trying to access non-existent tmpObj)
-			return nil
-		}
-		// Create random name to temporarily move file to
-		tmpObjName := dstFileName + "-rclone-move-" + random.String(8)
-		tmpObjFail, err := fdst.NewObject(ctx, tmpObjName)
-		if err != fs.ErrorObjectNotFound {
-			if err == nil {
-				logger(ctx, TransferError, nil, tmpObjFail, err)
-				return errors.New("found an already existing file with a randomly generated name. Try the operation again")
-			}
-			logger(ctx, TransferError, nil, tmpObjFail, err)
-			return fmt.Errorf("error while attempting to move file to a temporary location: %w", err)
-		}
-		tr := accounting.Stats(ctx).NewTransfer(srcObj)
+	if needsMoveCaseInsensitive(fdst, fsrc, dstFileName, srcFileName, cp) {
+		tr := accounting.Stats(ctx).NewTransfer(srcObj, fdst)
 		defer func() {
 			tr.Done(ctx, err)
 		}()
-		tmpObj, err := Op(ctx, fdst, nil, tmpObjName, srcObj)
-		if err != nil {
-			logger(ctx, TransferError, srcObj, tmpObj, err)
-			return fmt.Errorf("error while moving file to temporary location: %w", err)
-		}
-		_, err = Op(ctx, fdst, nil, dstFileName, tmpObj)
-		logger(ctx, MissingOnDst, tmpObj, nil, err)
+		_, err = moveCaseInsensitive(ctx, fdst, fsrc, dstFileName, srcFileName, cp, srcObj)
 		return err
 	}
 
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 39dfb3818..ddbe03a07 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -367,7 +367,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
 			}
 			// Fix case for case insensitive filesystems
 			if s.ci.FixCase && !s.ci.Immutable && src.Remote() != pair.Dst.Remote() {
-				if newDst, err := operations.Move(s.ctx, s.fdst, nil, src.Remote(), pair.Dst); err != nil {
+				if newDst, err := operations.Move(s.ctx, s.fdst, pair.Dst, src.Remote(), pair.Dst); err != nil {
 					fs.Errorf(pair.Dst, "Error while attempting to rename to %s: %v", src.Remote(), err)
 					s.processError(err)
 				} else {
@@ -1151,9 +1151,9 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
 			if s.ci.FixCase && !s.ci.Immutable && src.Remote() != dst.Remote() {
 				// Fix case for case insensitive filesystems
 				// Fix each dir before recursing into subdirs and files
-				oldDirFs, err := fs.NewFs(s.ctx, filepath.Join(s.fdst.Root(), dst.Remote()))
+				oldDirFs, err := fs.NewFs(s.ctx, filepath.Join(fs.ConfigStringFull(s.fdst), dst.Remote()))
 				s.processError(err)
-				newDirPath := filepath.Join(s.fdst.Root(), filepath.Dir(dst.Remote()), filepath.Base(src.Remote()))
+				newDirPath := filepath.Join(fs.ConfigStringFull(s.fdst), filepath.Dir(dst.Remote()), filepath.Base(src.Remote()))
 				newDirFs, err := fs.NewFs(s.ctx, newDirPath)
 				s.processError(err)
 				// Create random name to temporarily move dir to
diff --git a/lib/http/serve/dir.go b/lib/http/serve/dir.go
index 00d4579c5..1cfcc454b 100644
--- a/lib/http/serve/dir.go
+++ b/lib/http/serve/dir.go
@@ -224,7 +224,7 @@ const (
 // Serve serves a directory
 func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {
 	// Account the transfer
-	tr := accounting.Stats(r.Context()).NewTransferRemoteSize(d.DirRemote, -1)
+	tr := accounting.Stats(r.Context()).NewTransferRemoteSize(d.DirRemote, -1, nil, nil)
 	defer tr.Done(r.Context(), nil)
 
 	fs.Infof(d.DirRemote, "%s: Serving directory", r.RemoteAddr)
diff --git a/lib/http/serve/serve.go b/lib/http/serve/serve.go
index 6f97d0d8a..21f3c3322 100644
--- a/lib/http/serve/serve.go
+++ b/lib/http/serve/serve.go
@@ -79,7 +79,7 @@ func Object(w http.ResponseWriter, r *http.Request, o fs.Object) {
 		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
 		return
 	}
-	tr := accounting.Stats(r.Context()).NewTransfer(o)
+	tr := accounting.Stats(r.Context()).NewTransfer(o, nil)
 	defer func() {
 		tr.Done(r.Context(), err)
 	}()
diff --git a/vfs/read.go b/vfs/read.go
index 8bedeae91..f890143c8 100644
--- a/vfs/read.go
+++ b/vfs/read.go
@@ -79,7 +79,7 @@ func (fh *ReadFileHandle) openPending() (err error) {
 	if err != nil {
 		return err
 	}
-	tr := accounting.GlobalStats().NewTransfer(o)
+	tr := accounting.GlobalStats().NewTransfer(o, nil)
 	fh.done = tr.Done
 	fh.r = tr.Account(context.TODO(), r).WithBuffer() // account the transfer
 	fh.opened = true
diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.go
index b12c107b5..b0dd6b508 100644
--- a/vfs/vfscache/downloaders/downloaders.go
+++ b/vfs/vfscache/downloaders/downloaders.go
@@ -518,7 +518,7 @@ loop:
 // should be called on a fresh downloader
 func (dl *downloader) open(offset int64) (err error) {
 	// defer log.Trace(dl.dls.src, "offset=%d", offset)("err=%v", &err)
-	dl.tr = accounting.Stats(dl.dls.ctx).NewTransfer(dl.dls.src)
+	dl.tr = accounting.Stats(dl.dls.ctx).NewTransfer(dl.dls.src, nil)
 
 	size := dl.dls.src.Size()
 	if size < 0 {


Test patch:
diff --git a/fs/accounting/transfer_test.go b/fs/accounting/transfer_test.go
new file mode 100644
index 000000000..ec5c11e78
--- /dev/null
+++ b/fs/accounting/transfer_test.go
@@ -0,0 +1,66 @@
+package accounting
+
+import (
+	"context"
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/fstest/mockfs"
+	"github.com/rclone/rclone/fstest/mockobject"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTransfer(t *testing.T) {
+	ctx := context.Background()
+	s := NewStats(ctx)
+
+	o := mockobject.Object("obj")
+	srcFs, err := mockfs.NewFs(ctx, "srcFs", "srcFs", nil)
+	require.NoError(t, err)
+	dstFs, err := mockfs.NewFs(ctx, "dstFs", "dstFs", nil)
+	require.NoError(t, err)
+
+	tr := newTransfer(s, o, srcFs, dstFs)
+
+	t.Run("Snapshot", func(t *testing.T) {
+		snap := tr.Snapshot()
+		assert.Equal(t, "obj", snap.Name)
+		assert.Equal(t, int64(0), snap.Size)
+		assert.Equal(t, int64(0), snap.Bytes)
+		assert.Equal(t, false, snap.Checked)
+		assert.Equal(t, false, snap.StartedAt.IsZero())
+		assert.Equal(t, true, snap.CompletedAt.IsZero())
+		assert.Equal(t, nil, snap.Error)
+		assert.Equal(t, "", snap.Group)
+		assert.Equal(t, "srcFs:srcFs", snap.SrcFs)
+		assert.Equal(t, "dstFs:dstFs", snap.DstFs)
+	})
+
+	t.Run("Done", func(t *testing.T) {
+		tr.Done(ctx, io.EOF)
+		snap := tr.Snapshot()
+		assert.Equal(t, "obj", snap.Name)
+		assert.Equal(t, int64(0), snap.Size)
+		assert.Equal(t, int64(0), snap.Bytes)
+		assert.Equal(t, false, snap.Checked)
+		assert.Equal(t, false, snap.StartedAt.IsZero())
+		assert.Equal(t, false, snap.CompletedAt.IsZero())
+		assert.Equal(t, true, errors.Is(snap.Error, io.EOF))
+		assert.Equal(t, "", snap.Group)
+		assert.Equal(t, "srcFs:srcFs", snap.SrcFs)
+		assert.Equal(t, "dstFs:dstFs", snap.DstFs)
+	})
+
+	t.Run("rcStats", func(t *testing.T) {
+		out := tr.rcStats()
+		assert.Equal(t, rc.Params{
+			"name":  "obj",
+			"size":  int64(0),
+			"srcFs": "srcFs:srcFs",
+			"dstFs": "dstFs:dstFs",
+		}, out)
+	})
+}
diff --git a/fs/operations/check_test.go b/fs/operations/check_test.go
index 2f4213ad4..1ed2573cf 100644
--- a/fs/operations/check_test.go
+++ b/fs/operations/check_test.go
@@ -563,9 +563,6 @@ func TestApplyTransforms(t *testing.T) {
 
 	testScenario := func(checkfileName, remotefileName, scenario string) {
 		r := fstest.NewRunIndividual(t)
-		if !r.Flocal.Hashes().Contains(hashType) || !r.Fremote.Hashes().Contains(hashType) {
-			t.Skipf("Fs lacks %s, skipping", hashType)
-		}
 		ctx := context.Background()
 		ci := fs.GetConfig(ctx)
 		opt := operations.CheckOpt{}
@@ -580,7 +577,7 @@ func TestApplyTransforms(t *testing.T) {
 		ci.NoUnicodeNormalization = true
 		ci.IgnoreCaseSync = false
 		accounting.GlobalStats().ResetCounters()
-		err := operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, false)
+		err := operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, true)
 		assert.Error(t, err, "no expected error for %s %v %v", testname, checkfileName, remotefileName)
 
 		testname = scenario + " (with normalization)"
@@ -588,7 +585,7 @@ func TestApplyTransforms(t *testing.T) {
 		ci.NoUnicodeNormalization = false
 		ci.IgnoreCaseSync = true
 		accounting.GlobalStats().ResetCounters()
-		err = operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, false)
+		err = operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, true)
 		assert.NoError(t, err, "unexpected error for %s %v %v", testname, checkfileName, remotefileName)
 	}
 
diff --git a/fs/operations/multithread_test.go b/fs/operations/multithread_test.go
index a61f34331..59ff59d9d 100644
--- a/fs/operations/multithread_test.go
+++ b/fs/operations/multithread_test.go
@@ -183,7 +183,7 @@ func TestMultithreadCopy(t *testing.T) {
 				require.NoError(t, err)
 
 				accounting.GlobalStats().ResetCounters()
-				tr := accounting.GlobalStats().NewTransfer(src)
+				tr := accounting.GlobalStats().NewTransfer(src, nil)
 
 				defer func() {
 					tr.Done(ctx, err)
@@ -284,7 +284,7 @@ func TestMultithreadCopyAbort(t *testing.T) {
 	src, err := r.Flocal.NewObject(ctx, fileName)
 	require.NoError(t, err)
 	accounting.GlobalStats().ResetCounters()
-	tr := accounting.GlobalStats().NewTransfer(src)
+	tr := accounting.GlobalStats().NewTransfer(src, nil)
 
 	defer func() {
 		tr.Done(ctx, err)
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 325db1748..57cf76d27 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1005,10 +1005,10 @@ func TestCaseInsensitiveMoveFileDryRun(t *testing.T) {
 		return
 	}
 
-	ci.DryRun = true
 	file1 := r.WriteObject(ctx, "hello", "world", t1)
 	r.CheckRemoteItems(t, file1)
 
+	ci.DryRun = true
 	err := operations.MoveFile(ctx, r.Fremote, r.Fremote, "HELLO", file1.Path)
 	require.NoError(t, err)
 	r.CheckRemoteItems(t, file1)