Repository: rclone/rclone
Pull request #2981 (2019-02-18 06:05:00)

crypt: add option to not encrypt data #1077
#### What is the purpose of this change?

As discussed in #1077, I'm adding an option to leave the data unencrypted. This permits a user to have only the filenames encrypted if they so choose.

#### Was the change discussed in an issue or in the forum before?

#1077
https://forum.rclone.org/t/is-it-possible-to-encrpyt-filenames-only/902

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)


Vesnyx: Alright, sorry for the huge delay, been super busy recently. I'll be pushing the changes up as soon as I get back to my place.
ncw: > Alright, sorry for the huge delay, been super busy recently. I'll be pushing the changes up as soon as I get back to my place.

No worries.. Ping me here when you are done :-)
Vesnyx: Hey @ncw , I think I've finished up the majority of the changes. I also tried testing it locally and it appears to work and pass the tests. Let me know what else I need to do before it's ready to be added.
horganizzato: is this still being considered?
ncw: Sorry I've dropped the ball on this PR.

It needs rebasing against master and gofmt running on it, but it looks basically ready otherwise.
ivandeex: @Vesnyx Do you need a hand rebasing?
Max-Sum: Can I help rebasing this?
ncw: @Max-Sum wrote:
> Can I help rebasing this?

Yes please - that would be fantastic.

Vesnyx: Sorry everyone. Been swamped with work and haven't had a chance to work on this. I've just pushed the rebased commits. 
ivandeex: @Vesnyx 

I've pushed a commit addressing build errors.

```
golangci-lint run ./backend/crypt/...
backend/crypt/crypt.go:918:9: `if` block ends with a `return` statement, so drop this `else` and outdent its block (golint)
```
Just do what it wants.

`File is not goimportsed`
Just run `gofmt -w ./backend/crypt/crypt.go`

Please run this to obtain my commit: `cd /path/to/rclone; git fetch -v; git pull --ff-only`

Vesnyx: It has been done. :)
ivandeex: Rebased again due to recent breaking changes, to be sure.
ivandeex: @ncw please review
ivandeex: @ncw ping
ivandeex: @Vesnyx please address review comments
ivandeex: Rebased on current master
ivandeex: @ncw I have addressed your review comments. Please this PR. Thank you.

Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d8f4abebb..98adc0295 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -24,7 +24,7 @@ jobs:
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -34,7 +34,7 @@ jobs:
 
           - job_name: mac_amd64
             os: macOS-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -43,14 +43,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macOS-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -macos-sdk macosx11.1 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows_amd64
             os: windows-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             build_flags: '-include "^windows/amd64" -cgo'
             build_args: '-buildmode exe'
@@ -60,7 +60,7 @@ jobs:
 
           - job_name: windows_386
             os: windows-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             goarch: '386'
             cgo: '1'
@@ -71,7 +71,7 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go
index f3794e745..aa07b7a68 100644
--- a/backend/crypt/crypt.go
+++ b/backend/crypt/crypt.go
@@ -101,6 +101,21 @@ names, or for debugging purposes.`,
 			Default:  false,
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
+		}, {
+			Name:     "no_data_encryption",
+			Help:     "Option to either encrypt file data or leave it unencrypted.",
+			Default:  false,
+			Advanced: true,
+			Examples: []fs.OptionExample{
+				{
+					Value: "true",
+					Help:  "Don't encrypt file data, leave it unencrypted.",
+				},
+				{
+					Value: "false",
+					Help:  "Encrypt file data.",
+				},
+			},
 		}},
 	})
 }
@@ -209,6 +224,7 @@ type Options struct {
 	Remote                  string `config:"remote"`
 	FilenameEncryption      string `config:"filename_encryption"`
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
+	NoDataEncryption        bool   `config:"no_data_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
@@ -346,6 +362,10 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
+	if f.opt.NoDataEncryption {
+		return put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
+	}
+
 	// Encrypt the data into wrappedIn
 	wrappedIn, encrypter, err := f.cipher.encryptData(in)
 	if err != nil {
@@ -617,6 +637,10 @@ func (f *Fs) computeHashWithNonce(ctx context.Context, nonce nonce, src fs.Objec
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
+	if f.opt.NoDataEncryption {
+		return src.Hash(ctx, hashType)
+	}
+
 	// Read the nonce - opening the file is sufficient to read the nonce in
 	// use a limited read so we only read the header
 	in, err := o.Object.Open(ctx, &fs.RangeOption{Start: 0, End: int64(fileHeaderSize) - 1})
@@ -822,9 +846,13 @@ func (o *Object) Remote() string {
 
 // Size returns the size of the file
 func (o *Object) Size() int64 {
-	size, err := o.f.cipher.DecryptedSize(o.Object.Size())
-	if err != nil {
-		fs.Debugf(o, "Bad size for decrypt: %v", err)
+	size := o.Object.Size()
+	if !o.f.opt.NoDataEncryption {
+		var err error
+		size, err = o.f.cipher.DecryptedSize(size)
+		if err != nil {
+			fs.Debugf(o, "Bad size for decrypt: %v", err)
+		}
 	}
 	return size
 }
@@ -842,6 +870,10 @@ func (o *Object) UnWrap() fs.Object {
 
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
+	if o.f.opt.NoDataEncryption {
+		return o.Object.Open(ctx, options...)
+	}
+
 	var openOptions []fs.OpenOption
 	var offset, limit int64 = 0, -1
 	for _, option := range options {
diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go
index 0fab0081c..62e9784ed 100644
--- a/backend/ftp/ftp.go
+++ b/backend/ftp/ftp.go
@@ -93,6 +93,17 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Help:     "Disable using MLSD even if server advertises support",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:    "idle_timeout",
+			Default: fs.Duration(60 * time.Second),
+			Help: `Max time before closing idle connections
+
+If no connections have been returned to the connection pool in the time
+given, rclone will empty the connection pool.
+
+Set to 0 to keep connections indefinitely.
+`,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -120,6 +131,7 @@ type Options struct {
 	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
 	DisableEPSV       bool                 `config:"disable_epsv"`
 	DisableMLSD       bool                 `config:"disable_mlsd"`
+	IdleTimeout       fs.Duration          `config:"idle_timeout"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -136,6 +148,7 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
+	drain    *time.Timer // used to drain the pool when we stop using the connections
 	tokens   *pacer.TokenDispenser
 	tlsConf  *tls.Config
 }
@@ -322,9 +335,32 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	}
 	f.poolMu.Lock()
 	f.pool = append(f.pool, c)
+	if f.opt.IdleTimeout > 0 {
+		f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
+	}
 	f.poolMu.Unlock()
 }
 
+// Drain the pool of any connections
+func (f *Fs) drainPool(ctx context.Context) (err error) {
+	f.poolMu.Lock()
+	defer f.poolMu.Unlock()
+	if f.opt.IdleTimeout > 0 {
+		f.drain.Stop()
+	}
+	if len(f.pool) != 0 {
+		fs.Debugf(f, "closing %d unused connections", len(f.pool))
+	}
+	for i, c := range f.pool {
+		if cErr := c.Quit(); cErr != nil {
+			err = cErr
+		}
+		f.pool[i] = nil
+	}
+	f.pool = nil
+	return err
+}
+
 // NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
@@ -379,6 +415,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
+	// set the pool drainer timer going
+	if f.opt.IdleTimeout > 0 {
+		f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
+	}
 	// Make a connection and pool it to return errors early
 	c, err := f.getFtpConnection(ctx)
 	if err != nil {
@@ -407,6 +447,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	return f, err
 }
 
+// Shutdown the backend, closing any background tasks and any
+// cached connections.
+func (f *Fs) Shutdown(ctx context.Context) error {
+	return f.drainPool(ctx)
+}
+
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
 	switch errX := err.(type) {
@@ -1015,5 +1061,6 @@ var (
 	_ fs.Mover       = &Fs{}
 	_ fs.DirMover    = &Fs{}
 	_ fs.PutStreamer = &Fs{}
+	_ fs.Shutdowner  = &Fs{}
 	_ fs.Object      = &Object{}
 )
diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index f3519788a..c039aa045 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -204,6 +204,17 @@ Fstat instead of Stat which is called on an already open file handle.
 It has been found that this helps with IBM Sterling SFTP servers which have
 "extractability" level set to 1 which means only 1 file can be opened at
 any given time.
+`,
+			Advanced: true,
+		}, {
+			Name:    "idle_timeout",
+			Default: fs.Duration(60 * time.Second),
+			Help: `Max time before closing idle connections
+
+If no connections have been returned to the connection pool in the time
+given, rclone will empty the connection pool.
+
+Set to 0 to keep connections indefinitely.
 `,
 			Advanced: true,
 		}},
@@ -213,27 +224,28 @@ any given time.
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host              string `config:"host"`
-	User              string `config:"user"`
-	Port              string `config:"port"`
-	Pass              string `config:"pass"`
-	KeyPem            string `config:"key_pem"`
-	KeyFile           string `config:"key_file"`
-	KeyFilePass       string `config:"key_file_pass"`
-	PubKeyFile        string `config:"pubkey_file"`
-	KnownHostsFile    string `config:"known_hosts_file"`
-	KeyUseAgent       bool   `config:"key_use_agent"`
-	UseInsecureCipher bool   `config:"use_insecure_cipher"`
-	DisableHashCheck  bool   `config:"disable_hashcheck"`
-	AskPassword       bool   `config:"ask_password"`
-	PathOverride      string `config:"path_override"`
-	SetModTime        bool   `config:"set_modtime"`
-	Md5sumCommand     string `config:"md5sum_command"`
-	Sha1sumCommand    string `config:"sha1sum_command"`
-	SkipLinks         bool   `config:"skip_links"`
-	Subsystem         string `config:"subsystem"`
-	ServerCommand     string `config:"server_command"`
-	UseFstat          bool   `config:"use_fstat"`
+	Host              string      `config:"host"`
+	User              string      `config:"user"`
+	Port              string      `config:"port"`
+	Pass              string      `config:"pass"`
+	KeyPem            string      `config:"key_pem"`
+	KeyFile           string      `config:"key_file"`
+	KeyFilePass       string      `config:"key_file_pass"`
+	PubKeyFile        string      `config:"pubkey_file"`
+	KnownHostsFile    string      `config:"known_hosts_file"`
+	KeyUseAgent       bool        `config:"key_use_agent"`
+	UseInsecureCipher bool        `config:"use_insecure_cipher"`
+	DisableHashCheck  bool        `config:"disable_hashcheck"`
+	AskPassword       bool        `config:"ask_password"`
+	PathOverride      string      `config:"path_override"`
+	SetModTime        bool        `config:"set_modtime"`
+	Md5sumCommand     string      `config:"md5sum_command"`
+	Sha1sumCommand    string      `config:"sha1sum_command"`
+	SkipLinks         bool        `config:"skip_links"`
+	Subsystem         string      `config:"subsystem"`
+	ServerCommand     string      `config:"server_command"`
+	UseFstat          bool        `config:"use_fstat"`
+	IdleTimeout       fs.Duration `config:"idle_timeout"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -251,7 +263,8 @@ type Fs struct {
 	cachedHashes *hash.Set
 	poolMu       sync.Mutex
 	pool         []*conn
-	pacer        *fs.Pacer // pacer for operations
+	drain        *time.Timer // used to drain the pool when we stop using the connections
+	pacer        *fs.Pacer   // pacer for operations
 	savedpswd    string
 }
 
@@ -428,6 +441,9 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 	}
 	f.poolMu.Lock()
 	f.pool = append(f.pool, c)
+	if f.opt.IdleTimeout > 0 {
+		f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
+	}
 	f.poolMu.Unlock()
 }
 
@@ -435,6 +451,12 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 func (f *Fs) drainPool(ctx context.Context) (err error) {
 	f.poolMu.Lock()
 	defer f.poolMu.Unlock()
+	if f.opt.IdleTimeout > 0 {
+		f.drain.Stop()
+	}
+	if len(f.pool) != 0 {
+		fs.Debugf(f, "closing %d unused connections", len(f.pool))
+	}
 	for i, c := range f.pool {
 		if cErr := c.closed(); cErr == nil {
 			cErr = c.close()
@@ -667,6 +689,10 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 	f.mkdirLock = newStringLock()
 	f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
 	f.savedpswd = ""
+	// set the pool drainer timer going
+	if f.opt.IdleTimeout > 0 {
+		f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
+	}
 
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 340961e44..cdb67fa47 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -461,3 +461,4 @@ put them back in again.` >}}
   * David Sze <sze.david@gmail.com>
   * cynthia kwok <cynthia.m.kwok@gmail.com>
   * Ankur Gupta <agupta@egnyte.com>
+  * Miron Veryanskiy <MironVeryanskiy@gmail.com>
diff --git a/docs/content/cache.md b/docs/content/cache.md
index c2d6f7de9..c18ab19e1 100644
--- a/docs/content/cache.md
+++ b/docs/content/cache.md
@@ -205,7 +205,7 @@ These URLs are used by Plex internally to connect to the Plex server securely.
 
 The format for these URLs is the following:
 
-https://ip-with-dots-replaced.server-hash.plex.direct:32400/
+`https://ip-with-dots-replaced.server-hash.plex.direct:32400/`
 
 The `ip-with-dots-replaced` part can be any IPv4 address, where the dots
 have been replaced with dashes, e.g. `127.0.0.1` becomes `127-0-0-1`.
diff --git a/docs/content/filtering.md b/docs/content/filtering.md
index 4d2bd0250..4b81694c4 100644
--- a/docs/content/filtering.md
+++ b/docs/content/filtering.md
@@ -661,7 +661,7 @@ not list `dir3`, `file3` or `.ignore`.
 ## Common pitfalls
 
 The most frequent filter support issues on
-the [rclone forum](https://https://forum.rclone.org/) are:
+the [rclone forum](https://forum.rclone.org/) are:
 
 * Not using paths relative to the root of the remote
 * Not using `/` to match from the root of a remote
diff --git a/docs/content/sftp.md b/docs/content/sftp.md
index 172220011..eb1427362 100644
--- a/docs/content/sftp.md
+++ b/docs/content/sftp.md
@@ -526,8 +526,8 @@ The Go SSH library disables the use of the aes128-cbc cipher by
 default, due to security concerns. This can be re-enabled on a
 per-connection basis by setting the `use_insecure_cipher` setting in
 the configuration file to `true`. Further details on the insecurity of
-this cipher can be found [in this paper]
-(http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf).
+this cipher can be found
+[in this paper](http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf).
 
 SFTP isn't supported under plan9 until [this
 issue](https://github.com/pkg/sftp/issues/156) is fixed.
diff --git a/docs/content/zoho.md b/docs/content/zoho.md
index 3e76285a0..1c44cc3f2 100644
--- a/docs/content/zoho.md
+++ b/docs/content/zoho.md
@@ -76,11 +76,12 @@ y/e/d>
 See the [remote setup docs](/remote_setup/) for how to set it up on a
 machine with no Internet browser available.
 
-Note that rclone runs a webserver on your local machine to collect the
-token as returned from Zoho Workdrive. This only runs from the moment it
-opens your browser to the moment you get back the verification code.
-This is on `http://127.0.0.1:53682/` and this it may require you to
-unblock it temporarily if you are running a host firewall.
+Rclone runs a webserver on your local computer to collect the
+authorization token from Zoho Workdrive. This is only from the moment
+your browser is opened until the token is returned.
+The webserver runs on `http://127.0.0.1:53682/`.
+If local port `53682` is protected by a firewall you may need to temporarily
+unblock the firewall to complete authorization.
 
 Once configured you can then use `rclone` like this,
 
diff --git a/fs/cache/cache.go b/fs/cache/cache.go
index 106c0bbc5..dcc94adff 100644
--- a/fs/cache/cache.go
+++ b/fs/cache/cache.go
@@ -104,6 +104,19 @@ func Get(ctx context.Context, fsString string) (f fs.Fs, err error) {
 	return GetFn(ctx, fsString, fs.NewFs)
 }
 
+// GetArr gets []fs.Fs from []fsStrings either from the cache or creates it afresh
+func GetArr(ctx context.Context, fsStrings []string) (f []fs.Fs, err error) {
+	var fArr []fs.Fs
+	for _, fsString := range fsStrings {
+		f1, err1 := GetFn(ctx, fsString, fs.NewFs)
+		if err1 != nil {
+			return fArr, err1
+		}
+		fArr = append(fArr, f1)
+	}
+	return fArr, nil
+}
+
 // Put puts an fs.Fs named fsString into the cache
 func Put(fsString string, f fs.Fs) {
 	canonicalName := fs.ConfigString(f)
diff --git a/fs/config.go b/fs/config.go
index 8481bd4e0..fa7198e3b 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -76,8 +76,8 @@ type ConfigInfo struct {
 	NoUnicodeNormalization bool
 	NoUpdateModTime        bool
 	DataRateUnit           string
-	CompareDest            string
-	CopyDest               string
+	CompareDest            []string
+	CopyDest               []string
 	BackupDir              string
 	Suffix                 string
 	SuffixKeepExtension    bool
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index 0a4f177c7..b41c380e8 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -81,8 +81,8 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless.")
 	flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.")
 	flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination mod-time if files identical.")
-	flags.StringVarP(flagSet, &ci.CompareDest, "compare-dest", "", ci.CompareDest, "Include additional server-side path during comparison.")
-	flags.StringVarP(flagSet, &ci.CopyDest, "copy-dest", "", ci.CopyDest, "Implies --compare-dest but also copies files from path into destination.")
+	flags.StringArrayVarP(flagSet, &ci.CompareDest, "compare-dest", "", nil, "Include additional comma separated server-side paths during comparison.")
+	flags.StringArrayVarP(flagSet, &ci.CopyDest, "copy-dest", "", nil, "Implies --compare-dest but also copies files from paths into destination.")
 	flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR.")
 	flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files.")
 	flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix.")
@@ -217,7 +217,7 @@ func SetFlags(ci *fs.ConfigInfo) {
 		ci.DeleteMode = fs.DeleteModeDefault
 	}
 
-	if ci.CompareDest != "" && ci.CopyDest != "" {
+	if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {
 		log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
 	}
 
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 1b3cd69af..6e198c4bf 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -748,6 +748,16 @@ func SameConfig(fdst, fsrc fs.Info) bool {
 	return fdst.Name() == fsrc.Name()
 }
 
+// SameConfigArr returns true if any of []fsrcs has same config file entry with fdst
+func SameConfigArr(fdst fs.Info, fsrcs []fs.Fs) bool {
+	for _, fsrc := range fsrcs {
+		if fdst.Name() == fsrc.Name() {
+			return true
+		}
+	}
+	return false
+}
+
 // Same returns true if fdst and fsrc point to the same underlying Fs
 func Same(fdst, fsrc fs.Info) bool {
 	return SameConfig(fdst, fsrc) && strings.Trim(fdst.Root(), "/") == strings.Trim(fsrc.Root(), "/")
@@ -1354,9 +1364,9 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
 }
 
 // GetCompareDest sets up --compare-dest
-func GetCompareDest(ctx context.Context) (CompareDest fs.Fs, err error) {
+func GetCompareDest(ctx context.Context) (CompareDest []fs.Fs, err error) {
 	ci := fs.GetConfig(ctx)
-	CompareDest, err = cache.Get(ctx, ci.CompareDest)
+	CompareDest, err = cache.GetArr(ctx, ci.CompareDest)
 	if err != nil {
 		return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", ci.CompareDest, err))
 	}
@@ -1391,18 +1401,21 @@ func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (No
 }
 
 // GetCopyDest sets up --copy-dest
-func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest fs.Fs, err error) {
+func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest []fs.Fs, err error) {
 	ci := fs.GetConfig(ctx)
-	CopyDest, err = cache.Get(ctx, ci.CopyDest)
+	CopyDest, err = cache.GetArr(ctx, ci.CopyDest)
 	if err != nil {
 		return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", ci.CopyDest, err))
 	}
-	if !SameConfig(fdst, CopyDest) {
+	if !SameConfigArr(fdst, CopyDest) {
 		return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
 	}
-	if CopyDest.Features().Copy == nil {
-		return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server-side copy"))
+	for _, cf := range CopyDest {
+		if cf.Features().Copy == nil {
+			return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy"))
+		}
 	}
+
 	return CopyDest, nil
 }
 
@@ -1457,12 +1470,22 @@ func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, bac
 // does not need to be copied
 //
 // Returns True if src does not need to be copied
-func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
+func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest []fs.Fs, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
 	ci := fs.GetConfig(ctx)
-	if ci.CompareDest != "" {
-		return compareDest(ctx, dst, src, CompareOrCopyDest)
-	} else if ci.CopyDest != "" {
-		return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir)
+	if len(ci.CompareDest) > 0 {
+		for _, compareF := range CompareOrCopyDest {
+			NoNeedTransfer, err := compareDest(ctx, dst, src, compareF)
+			if NoNeedTransfer || err != nil {
+				return NoNeedTransfer, err
+			}
+		}
+	} else if len(ci.CopyDest) > 0 {
+		for _, copyF := range CompareOrCopyDest {
+			NoNeedTransfer, err := copyDest(ctx, fdst, dst, src, copyF, backupDir)
+			if NoNeedTransfer || err != nil {
+				return NoNeedTransfer, err
+			}
+		}
 	}
 	return false, nil
 }
@@ -1732,19 +1755,20 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
 		return err
 	}
 
-	var backupDir, copyDestDir fs.Fs
+	var backupDir fs.Fs
+	var copyDestDir []fs.Fs
 	if ci.BackupDir != "" || ci.Suffix != "" {
 		backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName)
 		if err != nil {
 			return errors.Wrap(err, "creating Fs for --backup-dir failed")
 		}
 	}
-	if ci.CompareDest != "" {
+	if len(ci.CompareDest) > 0 {
 		copyDestDir, err = GetCompareDest(ctx)
 		if err != nil {
 			return err
 		}
-	} else if ci.CopyDest != "" {
+	} else if len(ci.CopyDest) > 0 {
 		copyDestDir, err = GetCopyDest(ctx, fdst)
 		if err != nil {
 			return err
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 59cb7233e..e8ce10ad1 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -70,7 +70,7 @@ type syncCopyMove struct {
 	trackRenamesWg         sync.WaitGroup         // wg for background track renames
 	trackRenamesCh         chan fs.Object         // objects are pumped in here
 	renameCheck            []fs.Object            // accumulate files to check for rename here
-	compareCopyDest        fs.Fs                  // place to check for files to server-side copy
+	compareCopyDest        []fs.Fs                // place to check for files to server side copy
 	backupDir              fs.Fs                  // place to store overwrites/deletes
 	checkFirst             bool                   // if set run all the checkers before starting transfers
 }
@@ -212,13 +212,13 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 			return nil, err
 		}
 	}
-	if ci.CompareDest != "" {
+	if len(ci.CompareDest) > 0 {
 		var err error
 		s.compareCopyDest, err = operations.GetCompareDest(ctx)
 		if err != nil {
 			return nil, err
 		}
-	} else if ci.CopyDest != "" {
+	} else if len(ci.CopyDest) > 0 {
 		var err error
 		s.compareCopyDest, err = operations.GetCopyDest(ctx, fdst)
 		if err != nil {
@@ -890,7 +890,7 @@ func (s *syncCopyMove) run() error {
 	// Delete empty fsrc subdirectories
 	// if DoMove and --delete-empty-src-dirs flag is set
 	if s.DoMove && s.deleteEmptySrcDirs {
-		//delete empty subdirectories that were part of the move
+		// delete empty subdirectories that were part of the move
 		s.processError(s.deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs))
 	}
 
diff --git a/vfs/help.go b/vfs/help.go
index 67758f42e..b068f88e8 100644
--- a/vfs/help.go
+++ b/vfs/help.go
@@ -110,6 +110,13 @@ for two reasons.  Firstly because it is only checked every
 !--vfs-cache-poll-interval!.  Secondly because open files cannot be
 evicted from the cache.
 
+You **should not** run two copies of rclone using the same VFS cache
+with the same or overlapping remotes if using !--vfs-cache-mode > off!.
+This can potentially cause data corruption if you do. You can work
+around this by giving each rclone its own cache hierarchy with
+!--cache-dir!. You don't need to worry about this if the remotes in
+use don't overlap.
+
 #### --vfs-cache-mode off
 
 In this mode (the default) the cache will read directly from the remote and write


Test patch:
diff --git a/backend/crypt/crypt_test.go b/backend/crypt/crypt_test.go
index 144c7824f..24eaa2722 100644
--- a/backend/crypt/crypt_test.go
+++ b/backend/crypt/crypt_test.go
@@ -91,3 +91,26 @@ func TestObfuscate(t *testing.T) {
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
+
+// TestNoDataObfuscate runs integration tests against the remote
+func TestNoDataObfuscate(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
+	name := "TestCrypt4"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
+			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
+			{Name: name, Key: "no_data_encryption", Value: "true"},
+		},
+		SkipBadWindowsCharacters:     true,
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 796b49ed4..2421b5b9b 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -909,9 +909,9 @@ func TestCopyFileCompareDest(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 
-	ci.CompareDest = r.FremoteName + "/CompareDest"
+	ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
 	defer func() {
-		ci.CompareDest = ""
+		ci.CompareDest = nil
 	}()
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
 	require.NoError(t, err)
@@ -995,9 +995,9 @@ func TestCopyFileCopyDest(t *testing.T) {
 		t.Skip("Skipping test as remote does not support server-side copy")
 	}
 
-	ci.CopyDest = r.FremoteName + "/CopyDest"
+	ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
 	defer func() {
-		ci.CopyDest = ""
+		ci.CopyDest = nil
 	}()
 
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 37fb2abdf..3d7b54831 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -1480,9 +1480,9 @@ func TestSyncCompareDest(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 
-	ci.CompareDest = r.FremoteName + "/CompareDest"
+	ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
 	defer func() {
-		ci.CompareDest = ""
+		ci.CompareDest = []string{}
 	}()
 
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
@@ -1562,6 +1562,40 @@ func TestSyncCompareDest(t *testing.T) {
 	fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
 }
 
+// Test with multiple CompareDest
+func TestSyncMultipleCompareDest(t *testing.T) {
+	ctx := context.Background()
+	ci := fs.GetConfig(ctx)
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ci.CompareDest = []string{r.FremoteName + "/pre-dest1", r.FremoteName + "/pre-dest2"}
+	defer func() {
+		ci.CompareDest = []string{}
+	}()
+
+	// check empty dest, new compare
+	fsrc1 := r.WriteFile("1", "1", t1)
+	fsrc2 := r.WriteFile("2", "2", t1)
+	fsrc3 := r.WriteFile("3", "3", t1)
+	fstest.CheckItems(t, r.Flocal, fsrc1, fsrc2, fsrc3)
+
+	fdest1 := r.WriteObject(ctx, "pre-dest1/1", "1", t1)
+	fdest2 := r.WriteObject(ctx, "pre-dest2/2", "2", t1)
+	fstest.CheckItems(t, r.Fremote, fdest1, fdest2)
+
+	accounting.GlobalStats().ResetCounters()
+	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest")
+	require.NoError(t, err)
+	require.NoError(t, Sync(ctx, fdst, r.Flocal, false))
+
+	fdest3 := fsrc3
+	fdest3.Path = "dest/3"
+
+	fstest.CheckItems(t, fdst, fsrc3)
+	fstest.CheckItems(t, r.Fremote, fdest1, fdest2, fdest3)
+}
+
 // Test with CopyDest set
 func TestSyncCopyDest(t *testing.T) {
 	ctx := context.Background()
@@ -1573,9 +1607,9 @@ func TestSyncCopyDest(t *testing.T) {
 		t.Skip("Skipping test as remote does not support server-side copy")
 	}
 
-	ci.CopyDest = r.FremoteName + "/CopyDest"
+	ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
 	defer func() {
-		ci.CopyDest = ""
+		ci.CopyDest = []string{}
 	}()
 
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
diff --git a/fstest/testserver/init.d/docker.bash b/fstest/testserver/init.d/docker.bash
index 2bd00643d..442738986 100644
--- a/fstest/testserver/init.d/docker.bash
+++ b/fstest/testserver/init.d/docker.bash
@@ -2,13 +2,13 @@
 
 stop() {
     if status ; then
-        docker stop $NAME
+        docker stop "$NAME"
         echo "$NAME stopped"
     fi
 }
 
 status() {
-    if docker ps --format "{{.Names}}" | grep ^${NAME}$ >/dev/null ; then
+    if docker ps --format '{{.Names}}' | grep -q "^${NAME}$" ; then
         echo "$NAME running"
     else
         echo "$NAME not running"
@@ -18,5 +18,5 @@ status() {
 }
 
 docker_ip() {
-    docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $NAME    
+    docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{"\n"}}{{end}}' "$NAME" | head -1
 }
diff --git a/fstest/testserver/testserver.go b/fstest/testserver/testserver.go
index 4ec5e0acb..8026b6d6e 100644
--- a/fstest/testserver/testserver.go
+++ b/fstest/testserver/testserver.go
@@ -91,7 +91,7 @@ func start(name string) error {
 				continue
 			}
 
-			// fs.Debugf(name, "key = %q, envKey = %q, value = %q", key, envKey, value)
+			// fs.Debugf(name, "key = %q, envKey = %q, value = %q", key, envKey(name, string(key)), value)
 			err = os.Setenv(envKey(name, string(key)), string(value))
 			if err != nil {
 				return err