Repository: rclone/rclone
Pull request #7058 (2023-06-14 18:09:05)

docs: listremotes also includes remotes from env vars
"rclone listremotes" lists all the remotes in the config file and defined in environment variables. This is was clear from documentation.


Patch:
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index ec9756bb7..404b143c0 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -1063,11 +1063,7 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 				fs.Debugf(f, "Odd name received %q", remote)
 				continue
 			}
-			remote = remote[len(prefix):]
 			isDirectory := isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote)
-			if addContainer {
-				remote = path.Join(containerName, remote)
-			}
 			if isDirectory {
 				// Don't insert the root directory
 				if remote == directory {
@@ -1076,6 +1072,10 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 				// process directory markers as directories
 				remote = strings.TrimRight(remote, "/")
 			}
+			remote = remote[len(prefix):]
+			if addContainer {
+				remote = path.Join(containerName, remote)
+			}
 			// Send object
 			err = fn(remote, file, isDirectory)
 			if err != nil {
diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go
index c8537372d..4bd7a41e4 100644
--- a/backend/googlecloudstorage/googlecloudstorage.go
+++ b/backend/googlecloudstorage/googlecloudstorage.go
@@ -690,11 +690,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 				fs.Logf(f, "Odd name received %q", object.Name)
 				continue
 			}
-			remote = remote[len(prefix):]
 			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
-			if addBucket {
-				remote = path.Join(bucket, remote)
-			}
 			// is this a directory marker?
 			if isDirectory {
 				// Don't insert the root directory
@@ -704,6 +700,10 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 				// process directory markers as directories
 				remote = strings.TrimRight(remote, "/")
 			}
+			remote = remote[len(prefix):]
+			if addBucket {
+				remote = path.Join(bucket, remote)
+			}
 
 			err = fn(remote, object, isDirectory)
 			if err != nil {
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 0543d89cd..6a93af10a 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3727,11 +3727,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 				fs.Logf(f, "Odd name received %q", remote)
 				continue
 			}
-			remote = remote[len(opt.prefix):]
 			isDirectory := (remote == "" || strings.HasSuffix(remote, "/")) && object.Size != nil && *object.Size == 0
-			if opt.addBucket {
-				remote = bucket.Join(opt.bucket, remote)
-			}
 			// is this a directory marker?
 			if isDirectory {
 				if opt.noSkipMarkers {
@@ -3746,6 +3742,10 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 					remote = strings.TrimRight(remote, "/")
 				}
 			}
+			remote = remote[len(opt.prefix):]
+			if opt.addBucket {
+				remote = bucket.Join(opt.bucket, remote)
+			}
 			if versionIDs != nil {
 				err = fn(remote, object, versionIDs[i], isDirectory)
 			} else {
diff --git a/cmd/listremotes/listremotes.go b/cmd/listremotes/listremotes.go
index e473fcb21..46d540440 100644
--- a/cmd/listremotes/listremotes.go
+++ b/cmd/listremotes/listremotes.go
@@ -24,7 +24,7 @@ func init() {
 
 var commandDefinition = &cobra.Command{
 	Use:   "listremotes",
-	Short: `List all the remotes in the config file.`,
+	Short: `List all the remotes in the config file and defined in environment variables.`,
 	Long: `
 rclone listremotes lists all the available remotes from the config file.
 
diff --git a/cmd/mountlib/help.go b/cmd/mountlib/help.go
index 03ead5b7e..7e65d16d2 100644
--- a/cmd/mountlib/help.go
+++ b/cmd/mountlib/help.go
@@ -408,20 +408,19 @@ or create systemd mount units:
 |||
 # /etc/systemd/system/mnt-data.mount
 [Unit]
-After=network-online.target
+Description=Mount for /mnt/data
 [Mount]
 Type=rclone
 What=sftp1:subdir
 Where=/mnt/data
-Options=rw,allow_other,args2env,vfs-cache-mode=writes,config=/etc/rclone.conf,cache-dir=/var/rclone
+Options=rw,_netdev,allow_other,args2env,vfs-cache-mode=writes,config=/etc/rclone.conf,cache-dir=/var/rclone
 |||
 
 optionally accompanied by systemd automount unit
 |||
 # /etc/systemd/system/mnt-data.automount
 [Unit]
-After=network-online.target
-Before=remote-fs.target
+Description=AutoMount for /mnt/data
 [Automount]
 Where=/mnt/data
 TimeoutIdleSec=600
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 2360f3cf2..4c0ad378e 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -729,3 +729,6 @@ put them back in again.` >}}
   * Tareq Sharafy <tareq.sha@gmail.com>
   * kapitainsky <dariuszb@me.com>
   * douchen <playgoobug@gmail.com>
+  * Sam Lai <70988+slai@users.noreply.github.com>
+  * URenko <18209292+URenko@users.noreply.github.com>
+  * Stanislav Gromov <kullfar@gmail.com>
diff --git a/docs/content/commands/rclone_listremotes.md b/docs/content/commands/rclone_listremotes.md
index fdf20b88c..c067b790e 100644
--- a/docs/content/commands/rclone_listremotes.md
+++ b/docs/content/commands/rclone_listremotes.md
@@ -1,6 +1,6 @@
 ---
 title: "rclone listremotes"
-description: "List all the remotes in the config file."
+description: "List all the remotes in the config file and defined in environment variables."
 slug: rclone_listremotes
 url: /commands/rclone_listremotes/
 versionIntroduced: v1.34
diff --git a/docs/content/commands/rclone_mount.md b/docs/content/commands/rclone_mount.md
index 322a3426c..37583f110 100644
--- a/docs/content/commands/rclone_mount.md
+++ b/docs/content/commands/rclone_mount.md
@@ -407,8 +407,6 @@ mount sftp1:subdir /mnt/data -t rclone -o vfs_cache_mode=writes,sftp_key_file=/p
 or create systemd mount units:
 ```
 # /etc/systemd/system/mnt-data.mount
-[Unit]
-After=network-online.target
 [Mount]
 Type=rclone
 What=sftp1:subdir
@@ -420,7 +418,6 @@ optionally accompanied by systemd automount unit
 ```
 # /etc/systemd/system/mnt-data.automount
 [Unit]
-After=network-online.target
 Before=remote-fs.target
 [Automount]
 Where=/mnt/data
diff --git a/docs/content/docs.md b/docs/content/docs.md
index fe68b65b7..8dc82d96e 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -1511,6 +1511,25 @@ if you are reading and writing to an OS X filing system this will be
 
 This command line flag allows you to override that computed default.
 
+### --multi-thread-write-buffer-size=SIZE ###
+
+When downloading with multiple threads, rclone will buffer SIZE bytes in
+memory before writing to disk for each thread.
+
+This can improve performance if the underlying filesystem does not deal
+well with a lot of small writes in different positions of the file, so
+if you see downloads being limited by disk write speed, you might want
+to experiment with different values. Specially for magnetic drives and
+remote file systems a higher value can be useful.
+
+Nevertheless, the default of `128k` should be fine for almost all use
+cases, so before changing it ensure that network is not really your
+bottleneck.
+
+As a final hint, size is not the only factor: block size (or similar
+concept) can have an impact. In one case, we observed that exact
+multiples of 16k performed much better than other values.
+
 ### --multi-thread-cutoff=SIZE ###
 
 When downloading files to the local backend above this size, rclone
diff --git a/docs/content/drive.md b/docs/content/drive.md
index c687315e9..12d9b0545 100644
--- a/docs/content/drive.md
+++ b/docs/content/drive.md
@@ -403,7 +403,7 @@ like a symlink in unix, except they point to the underlying file data
 (e.g. the inode in unix terms) so they don't break if the source is
 renamed or moved about.
 
-Be default rclone treats these as follows.
+By default rclone treats these as follows.
 
 For shortcuts pointing to files:
 
diff --git a/fs/accounting/accounting_unix.go b/fs/accounting/accounting_unix.go
index ec79ce544..761466a55 100644
--- a/fs/accounting/accounting_unix.go
+++ b/fs/accounting/accounting_unix.go
@@ -23,15 +23,26 @@ func (tb *tokenBucket) startSignalHandler() {
 		// This runs forever, but blocks until the signal is received.
 		for {
 			<-signals
-			tb.mu.Lock()
-			tb.toggledOff = !tb.toggledOff
-			tb.curr, tb.prev = tb.prev, tb.curr
-			s := "disabled"
-			if !tb.curr._isOff() {
-				s = "enabled"
-			}
-			tb.mu.Unlock()
-			fs.Logf(nil, "Bandwidth limit %s by user", s)
+
+			func() {
+				tb.mu.Lock()
+				defer tb.mu.Unlock()
+
+				// if there's no bandwidth limit configured now, do nothing
+				if !tb.currLimit.Bandwidth.IsSet() {
+					fs.Debugf(nil, "SIGUSR2 received but no bandwidth limit configured right now, ignoring")
+					return
+				}
+
+				tb.toggledOff = !tb.toggledOff
+				tb.curr, tb.prev = tb.prev, tb.curr
+				s := "disabled"
+				if !tb.curr._isOff() {
+					s = "enabled"
+				}
+
+				fs.Logf(nil, "Bandwidth limit %s by user", s)
+			}()
 		}
 	}()
 }
diff --git a/fs/accounting/token_bucket.go b/fs/accounting/token_bucket.go
index c875c568f..29204b5eb 100644
--- a/fs/accounting/token_bucket.go
+++ b/fs/accounting/token_bucket.go
@@ -30,12 +30,11 @@ type buckets [TokenBucketSlots]*rate.Limiter
 
 // tokenBucket holds info about the rate limiters in use
 type tokenBucket struct {
-	mu          sync.RWMutex // protects the token bucket variables
-	curr        buckets
-	prev        buckets
-	toggledOff  bool
-	currLimitMu sync.Mutex // protects changes to the timeslot
-	currLimit   fs.BwTimeSlot
+	mu         sync.RWMutex // protects the token bucket variables
+	curr       buckets
+	prev       buckets
+	toggledOff bool
+	currLimit  fs.BwTimeSlot
 }
 
 // Return true if limit is disabled
@@ -106,11 +105,11 @@ func (tb *tokenBucket) StartTokenBucket(ctx context.Context) {
 	if tb.currLimit.Bandwidth.IsSet() {
 		tb.curr = newTokenBucket(tb.currLimit.Bandwidth)
 		fs.Infof(nil, "Starting bandwidth limiter at %v Byte/s", &tb.currLimit.Bandwidth)
-
-		// Start the SIGUSR2 signal handler to toggle bandwidth.
-		// This function does nothing in windows systems.
-		tb.startSignalHandler()
 	}
+
+	// Start the SIGUSR2 signal handler to toggle bandwidth.
+	// This function does nothing in windows systems.
+	tb.startSignalHandler()
 }
 
 // StartTokenTicker creates a ticker to update the bandwidth limiter every minute.
@@ -126,11 +125,9 @@ func (tb *tokenBucket) StartTokenTicker(ctx context.Context) {
 	go func() {
 		for range ticker.C {
 			limitNow := ci.BwLimit.LimitAt(time.Now())
-			tb.currLimitMu.Lock()
+			tb.mu.Lock()
 
 			if tb.currLimit.Bandwidth != limitNow.Bandwidth {
-				tb.mu.Lock()
-
 				// If bwlimit is toggled off, the change should only
 				// become active on the next toggle, which causes
 				// an exchange of tb.curr <-> tb.prev
@@ -156,9 +153,9 @@ func (tb *tokenBucket) StartTokenTicker(ctx context.Context) {
 				}
 
 				tb.currLimit = limitNow
-				tb.mu.Unlock()
 			}
-			tb.currLimitMu.Unlock()
+
+			tb.mu.Unlock()
 		}
 	}()
 }
diff --git a/fs/config.go b/fs/config.go
index a6de70b85..060567447 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -51,101 +51,102 @@ var (
 
 // ConfigInfo is filesystem config options
 type ConfigInfo struct {
-	LogLevel                LogLevel
-	StatsLogLevel           LogLevel
-	UseJSONLog              bool
-	DryRun                  bool
-	Interactive             bool
-	CheckSum                bool
-	SizeOnly                bool
-	IgnoreTimes             bool
-	IgnoreExisting          bool
-	IgnoreErrors            bool
-	ModifyWindow            time.Duration
-	Checkers                int
-	Transfers               int
-	ConnectTimeout          time.Duration // Connect timeout
-	Timeout                 time.Duration // Data channel timeout
-	ExpectContinueTimeout   time.Duration
-	Dump                    DumpFlags
-	InsecureSkipVerify      bool // Skip server certificate verification
-	DeleteMode              DeleteMode
-	MaxDelete               int64
-	MaxDeleteSize           SizeSuffix
-	TrackRenames            bool   // Track file renames.
-	TrackRenamesStrategy    string // Comma separated list of strategies used to track renames
-	LowLevelRetries         int
-	UpdateOlder             bool // Skip files that are newer on the destination
-	NoGzip                  bool // Disable compression
-	MaxDepth                int
-	IgnoreSize              bool
-	IgnoreChecksum          bool
-	IgnoreCaseSync          bool
-	NoTraverse              bool
-	CheckFirst              bool
-	NoCheckDest             bool
-	NoUnicodeNormalization  bool
-	NoUpdateModTime         bool
-	DataRateUnit            string
-	CompareDest             []string
-	CopyDest                []string
-	BackupDir               string
-	Suffix                  string
-	SuffixKeepExtension     bool
-	UseListR                bool
-	BufferSize              SizeSuffix
-	BwLimit                 BwTimetable
-	BwLimitFile             BwTimetable
-	TPSLimit                float64
-	TPSLimitBurst           int
-	BindAddr                net.IP
-	DisableFeatures         []string
-	UserAgent               string
-	Immutable               bool
-	AutoConfirm             bool
-	StreamingUploadCutoff   SizeSuffix
-	StatsFileNameLength     int
-	AskPassword             bool
-	PasswordCommand         SpaceSepList
-	UseServerModTime        bool
-	MaxTransfer             SizeSuffix
-	MaxDuration             time.Duration
-	CutoffMode              CutoffMode
-	MaxBacklog              int
-	MaxStatsGroups          int
-	StatsOneLine            bool
-	StatsOneLineDate        bool   // If we want a date prefix at all
-	StatsOneLineDateFormat  string // If we want to customize the prefix
-	ErrorOnNoTransfer       bool   // Set appropriate exit code if no files transferred
-	Progress                bool
-	ProgressTerminalTitle   bool
-	Cookie                  bool
-	UseMmap                 bool
-	CaCert                  []string // Client Side CA
-	ClientCert              string   // Client Side Cert
-	ClientKey               string   // Client Side Key
-	MultiThreadCutoff       SizeSuffix
-	MultiThreadStreams      int
-	MultiThreadSet          bool   // whether MultiThreadStreams was set (set in fs/config/configflags)
-	OrderBy                 string // instructions on how to order the transfer
-	UploadHeaders           []*HTTPOption
-	DownloadHeaders         []*HTTPOption
-	Headers                 []*HTTPOption
-	MetadataSet             Metadata // extra metadata to write when uploading
-	RefreshTimes            bool
-	NoConsole               bool
-	TrafficClass            uint8
-	FsCacheExpireDuration   time.Duration
-	FsCacheExpireInterval   time.Duration
-	DisableHTTP2            bool
-	HumanReadable           bool
-	KvLockTime              time.Duration // maximum time to keep key-value database locked by process
-	DisableHTTPKeepAlives   bool
-	Metadata                bool
-	ServerSideAcrossConfigs bool
-	TerminalColorMode       TerminalColorMode
-	DefaultTime             Time // time that directories with no time should display
-	Inplace                 bool // Download directly to destination file instead of atomic download to temp/rename
+	LogLevel                   LogLevel
+	StatsLogLevel              LogLevel
+	UseJSONLog                 bool
+	DryRun                     bool
+	Interactive                bool
+	CheckSum                   bool
+	SizeOnly                   bool
+	IgnoreTimes                bool
+	IgnoreExisting             bool
+	IgnoreErrors               bool
+	ModifyWindow               time.Duration
+	Checkers                   int
+	Transfers                  int
+	ConnectTimeout             time.Duration // Connect timeout
+	Timeout                    time.Duration // Data channel timeout
+	ExpectContinueTimeout      time.Duration
+	Dump                       DumpFlags
+	InsecureSkipVerify         bool // Skip server certificate verification
+	DeleteMode                 DeleteMode
+	MaxDelete                  int64
+	MaxDeleteSize              SizeSuffix
+	TrackRenames               bool   // Track file renames.
+	TrackRenamesStrategy       string // Comma separated list of strategies used to track renames
+	LowLevelRetries            int
+	UpdateOlder                bool // Skip files that are newer on the destination
+	NoGzip                     bool // Disable compression
+	MaxDepth                   int
+	IgnoreSize                 bool
+	IgnoreChecksum             bool
+	IgnoreCaseSync             bool
+	NoTraverse                 bool
+	CheckFirst                 bool
+	NoCheckDest                bool
+	NoUnicodeNormalization     bool
+	NoUpdateModTime            bool
+	DataRateUnit               string
+	CompareDest                []string
+	CopyDest                   []string
+	BackupDir                  string
+	Suffix                     string
+	SuffixKeepExtension        bool
+	UseListR                   bool
+	BufferSize                 SizeSuffix
+	MultiThreadWriteBufferSize SizeSuffix
+	BwLimit                    BwTimetable
+	BwLimitFile                BwTimetable
+	TPSLimit                   float64
+	TPSLimitBurst              int
+	BindAddr                   net.IP
+	DisableFeatures            []string
+	UserAgent                  string
+	Immutable                  bool
+	AutoConfirm                bool
+	StreamingUploadCutoff      SizeSuffix
+	StatsFileNameLength        int
+	AskPassword                bool
+	PasswordCommand            SpaceSepList
+	UseServerModTime           bool
+	MaxTransfer                SizeSuffix
+	MaxDuration                time.Duration
+	CutoffMode                 CutoffMode
+	MaxBacklog                 int
+	MaxStatsGroups             int
+	StatsOneLine               bool
+	StatsOneLineDate           bool   // If we want a date prefix at all
+	StatsOneLineDateFormat     string // If we want to customize the prefix
+	ErrorOnNoTransfer          bool   // Set appropriate exit code if no files transferred
+	Progress                   bool
+	ProgressTerminalTitle      bool
+	Cookie                     bool
+	UseMmap                    bool
+	CaCert                     []string // Client Side CA
+	ClientCert                 string   // Client Side Cert
+	ClientKey                  string   // Client Side Key
+	MultiThreadCutoff          SizeSuffix
+	MultiThreadStreams         int
+	MultiThreadSet             bool   // whether MultiThreadStreams was set (set in fs/config/configflags)
+	OrderBy                    string // instructions on how to order the transfer
+	UploadHeaders              []*HTTPOption
+	DownloadHeaders            []*HTTPOption
+	Headers                    []*HTTPOption
+	MetadataSet                Metadata // extra metadata to write when uploading
+	RefreshTimes               bool
+	NoConsole                  bool
+	TrafficClass               uint8
+	FsCacheExpireDuration      time.Duration
+	FsCacheExpireInterval      time.Duration
+	DisableHTTP2               bool
+	HumanReadable              bool
+	KvLockTime                 time.Duration // maximum time to keep key-value database locked by process
+	DisableHTTPKeepAlives      bool
+	Metadata                   bool
+	ServerSideAcrossConfigs    bool
+	TerminalColorMode          TerminalColorMode
+	DefaultTime                Time // time that directories with no time should display
+	Inplace                    bool // Download directly to destination file instead of atomic download to temp/rename
 }
 
 // NewConfig creates a new config with everything set to the default
@@ -170,6 +171,7 @@ func NewConfig() *ConfigInfo {
 	c.MaxDepth = -1
 	c.DataRateUnit = "bytes"
 	c.BufferSize = SizeSuffix(16 << 20)
+	c.MultiThreadWriteBufferSize = SizeSuffix(128 * 1024)
 	c.UserAgent = "rclone/" + Version
 	c.StreamingUploadCutoff = SizeSuffix(100 * 1024)
 	c.MaxStatsGroups = 1000
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index 60047227e..a3eeedf70 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -126,6 +126,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.StringVarP(flagSet, &ci.ClientKey, "client-key", "", ci.ClientKey, "Client SSL private key (PEM) for mutual TLS auth")
 	flags.FVarP(flagSet, &ci.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size")
 	flags.IntVarP(flagSet, &ci.MultiThreadStreams, "multi-thread-streams", "", ci.MultiThreadStreams, "Max number of streams to use for multi-thread downloads")
+	flags.FVarP(flagSet, &ci.MultiThreadWriteBufferSize, "multi-thread-write-buffer-size", "", "In memory buffer size for writing when in multi-thread mode")
 	flags.BoolVarP(flagSet, &ci.UseJSONLog, "use-json-log", "", ci.UseJSONLog, "Use json log format")
 	flags.StringVarP(flagSet, &ci.OrderBy, "order-by", "", ci.OrderBy, "Instructions on how to order the transfers, e.g. 'size,descending'")
 	flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions")
diff --git a/fs/newfs.go b/fs/newfs.go
index 52375e86e..9be890ec7 100644
--- a/fs/newfs.go
+++ b/fs/newfs.go
@@ -115,11 +115,11 @@ func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, conne
 	return fsInfo, configName, fsPath, parsed.Config, err
 }
 
-// ConfigString returns a canonical version of the config string used
+// configString returns a canonical version of the config string used
 // to configure the Fs as passed to fs.NewFs
-func ConfigString(f Fs) string {
+func configString(f Fs, full bool) string {
 	name := f.Name()
-	if open := strings.IndexRune(name, '{'); open >= 0 && strings.HasSuffix(name, "}") {
+	if open := strings.IndexRune(name, '{'); full && open >= 0 && strings.HasSuffix(name, "}") {
 		suffix := name[open:]
 		overriddenConfigMu.Lock()
 		config, ok := overriddenConfig[suffix]
@@ -137,6 +137,21 @@ func ConfigString(f Fs) string {
 	return name + ":" + root
 }
 
+// ConfigString returns a canonical version of the config string used
+// to configure the Fs as passed to fs.NewFs. For Fs with extra
+// parameters this will include a canonical {hexstring} suffix.
+func ConfigString(f Fs) string {
+	return configString(f, false)
+}
+
+// ConfigStringFull returns a canonical version of the config string
+// used to configure the Fs as passed to fs.NewFs. This string can be
+// used to re-instantiate the Fs exactly so includes all the extra
+// parameters passed in.
+func ConfigStringFull(f Fs) string {
+	return configString(f, true)
+}
+
 // TemporaryLocalFs creates a local FS in the OS's temporary directory.
 //
 // No cleanup is performed, the caller must call Purge on the Fs themselves.
diff --git a/fs/operations/multithread.go b/fs/operations/multithread.go
index b4b79a5a6..5ab41b9a2 100644
--- a/fs/operations/multithread.go
+++ b/fs/operations/multithread.go
@@ -1,6 +1,7 @@
 package operations
 
 import (
+	"bufio"
 	"context"
 	"errors"
 	"fmt"
@@ -12,11 +13,32 @@ import (
 )
 
 const (
-	multithreadChunkSize     = 64 << 10
-	multithreadChunkSizeMask = multithreadChunkSize - 1
-	multithreadBufferSize    = 32 * 1024
+	multithreadChunkSize      = 64 << 10
+	multithreadChunkSizeMask  = multithreadChunkSize - 1
+	multithreadReadBufferSize = 32 * 1024
 )
 
+// An offsetWriter maps writes at offset base to offset base+off in the underlying writer.
+//
+// Modified from the go source code. Can be replaced with
+// io.OffsetWriter when we no longer need to support go1.19
+type offsetWriter struct {
+	w   io.WriterAt
+	off int64 // the current offset
+}
+
+// newOffsetWriter returns an offsetWriter that writes to w
+// starting at offset off.
+func newOffsetWriter(w io.WriterAt, off int64) *offsetWriter {
+	return &offsetWriter{w, off}
+}
+
+func (o *offsetWriter) Write(p []byte) (n int, err error) {
+	n, err = o.w.WriteAt(p, o.off)
+	o.off += int64(n)
+	return
+}
+
 // Return a boolean as to whether we should use multi thread copy for
 // this transfer
 func doMultiThreadCopy(ctx context.Context, f fs.Fs, src fs.Object) bool {
@@ -62,6 +84,7 @@ type multiThreadCopyState struct {
 
 // Copy a single stream into place
 func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int) (err error) {
+	ci := fs.GetConfig(ctx)
 	defer func() {
 		if err != nil {
 			fs.Debugf(mc.src, "multi-thread copy: stream %d/%d failed: %v", stream+1, mc.streams, err)
@@ -84,8 +107,13 @@ func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int) (err
 	}
 	defer fs.CheckClose(rc, &err)
 
+	var writer io.Writer = newOffsetWriter(mc.wc, start)
+	if ci.MultiThreadWriteBufferSize > 0 {
+		writer = bufio.NewWriterSize(writer, int(ci.MultiThreadWriteBufferSize))
+		fs.Debugf(mc.src, "multi-thread copy: write buffer set to %v", ci.MultiThreadWriteBufferSize)
+	}
 	// Copy the data
-	buf := make([]byte, multithreadBufferSize)
+	buf := make([]byte, multithreadReadBufferSize)
 	offset := start
 	for {
 		// Check if context cancelled and exit if so
@@ -98,7 +126,7 @@ func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int) (err
 			if err != nil {
 				return fmt.Errorf("multipart copy: accounting failed: %w", err)
 			}
-			nw, ew := mc.wc.WriteAt(buf[0:nr], offset)
+			nw, ew := writer.Write(buf[0:nr])
 			if nw > 0 {
 				offset += int64(nw)
 			}
@@ -113,6 +141,16 @@ func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int) (err
 			if er != io.EOF {
 				return fmt.Errorf("multipart copy: read failed: %w", er)
 			}
+
+			// if we were buffering, flush do disk
+			switch w := writer.(type) {
+			case *bufio.Writer:
+				er2 := w.Flush()
+				if er2 != nil {
+					return fmt.Errorf("multipart copy: flush failed: %w", er2)
+				}
+			}
+
 			break
 		}
 	}
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 4308df7fa..8c20fbe40 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -529,6 +529,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 	if err != nil {
 		err = fs.CountError(err)
 		fs.Errorf(src, "Failed to copy: %v", err)
+		if !inplace {
+			removeFailedPartialCopy(ctx, f, remotePartial)
+		}
 		return newDst, err
 	}
 


Test patch:
diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go
index d4b4e1e17..f48045072 100644
--- a/backend/s3/s3_internal_test.go
+++ b/backend/s3/s3_internal_test.go
@@ -318,7 +318,7 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 
 		// Check we can make a NewFs from that object with a version suffix
 		t.Run("NewFs", func(t *testing.T) {
-			newPath := bucket.Join(fs.ConfigString(f), fileNameVersion)
+			newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion)
 			// Make sure --s3-versions is set in the config of the new remote
 			fs.Debugf(nil, "oldPath = %q", newPath)
 			lastColon := strings.LastIndex(newPath, ":")
@@ -330,7 +330,7 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 			require.Equal(t, fs.ErrorIsFile, err)
 			require.NotNil(t, fNew)
 			// With the directory the directory above
-			assert.Equal(t, dirName, path.Base(fs.ConfigString(fNew)))
+			assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew)))
 		})
 	})
 
diff --git a/fs/newfs_test.go b/fs/newfs_test.go
index 3f86cd2d5..4d5ef8520 100644
--- a/fs/newfs_test.go
+++ b/fs/newfs_test.go
@@ -32,12 +32,14 @@ func TestNewFs(t *testing.T) {
 	assert.Equal(t, ":mockfs{S_NHG}", f2.Name())
 	assert.Equal(t, "/tmp", f2.Root())
 
-	assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigString(f2))
+	assert.Equal(t, ":mockfs{S_NHG}:/tmp", fs.ConfigString(f2))
+	assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigStringFull(f2))
 
 	f3, err := fs.NewFs(ctx, ":mockfs,potato='true':/tmp")
 	require.NoError(t, err)
 	assert.Equal(t, ":mockfs{S_NHG}", f3.Name())
 	assert.Equal(t, "/tmp", f3.Root())
 
-	assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigString(f3))
+	assert.Equal(t, ":mockfs{S_NHG}:/tmp", fs.ConfigString(f3))
+	assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigStringFull(f3))
 }