Repository: rclone/rclone
Pull request #4648 (2020-10-03 08:46:29)

s3: Add missing regions for AWS
Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?

Added missing regions for AWS S3

#### Was the change discussed in an issue or in the forum before?

https://forum.rclone.org/t/please-add-af-south-1-region-to-rclone-config/19512

#### Checklist

- [X] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [X] I have added tests for all changes in this PR if appropriate.
- [X] I have added documentation for the changes if appropriate.
- [X] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [X] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go
index 2f80b336c..7358d3174 100644
--- a/backend/jottacloud/jottacloud.go
+++ b/backend/jottacloud/jottacloud.go
@@ -1087,8 +1087,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		retry, _ := shouldRetry(resp, err)
-		return (retry && resp.StatusCode != 500), err
+		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1192,18 +1191,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
 
-	// surprise! jottacloud fucked up dirmove - the api spits out an error but
-	// dir gets moved regardless
-	if apiErr, ok := err.(*api.Error); ok {
-		if apiErr.StatusCode == 500 {
-			_, err := f.NewObject(ctx, dstRemote)
-			if err == fs.ErrorNotAFile {
-				log.Printf("FIXME: ignoring DirMove error - move succeeded anyway\n")
-				return nil
-			}
-			return err
-		}
-	}
 	if err != nil {
 		return errors.Wrap(err, "couldn't move directory")
 	}
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 14255de31..55f66596c 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -122,6 +122,9 @@ func init() {
 			Name: "secret_access_key",
 			Help: "AWS Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.",
 		}, {
+			// References:
+			// 1. https://docs.aws.amazon.com/general/latest/gr/rande.html
+			// 2. https://docs.aws.amazon.com/general/latest/gr/s3.html
 			Name:     "region",
 			Help:     "Region to connect to.",
 			Provider: "AWS",
@@ -131,12 +134,12 @@ func init() {
 			}, {
 				Value: "us-east-2",
 				Help:  "US East (Ohio) Region\nNeeds location constraint us-east-2.",
-			}, {
-				Value: "us-west-2",
-				Help:  "US West (Oregon) Region\nNeeds location constraint us-west-2.",
 			}, {
 				Value: "us-west-1",
 				Help:  "US West (Northern California) Region\nNeeds location constraint us-west-1.",
+			}, {
+				Value: "us-west-2",
+				Help:  "US West (Oregon) Region\nNeeds location constraint us-west-2.",
 			}, {
 				Value: "ca-central-1",
 				Help:  "Canada (Central) Region\nNeeds location constraint ca-central-1.",
@@ -146,9 +149,15 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region\nNeeds location constraint eu-west-2.",
+			}, {
+				Value: "eu-west-3",
+				Help:  "EU (Paris) Region\nNeeds location constraint eu-west-3.",
 			}, {
 				Value: "eu-north-1",
 				Help:  "EU (Stockholm) Region\nNeeds location constraint eu-north-1.",
+			}, {
+				Value: "eu-south-1",
+				Help:  "EU (Milan) Region\nNeeds location constraint eu-south-1.",
 			}, {
 				Value: "eu-central-1",
 				Help:  "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
@@ -164,6 +173,9 @@ func init() {
 			}, {
 				Value: "ap-northeast-2",
 				Help:  "Asia Pacific (Seoul)\nNeeds location constraint ap-northeast-2.",
+			}, {
+				Value: "ap-northeast-3",
+				Help:  "Asia Pacific (Osaka-Local)\nNeeds location constraint ap-northeast-3.",
 			}, {
 				Value: "ap-south-1",
 				Help:  "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.",
@@ -173,6 +185,24 @@ func init() {
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
+			}, {
+				Value: "me-south-1",
+				Help:  "Middle East (Bahrain) Region\nNeeds location constraint me-south-1.",
+			}, {
+				Value: "af-south-1",
+				Help:  "Africa (Cape Town) Region\nNeeds location constraint af-south-1.",
+			}, {
+				Value: "cn-north-1",
+				Help:  "China (Beijing) Region\nNeeds location constraint cn-north-1.",
+			}, {
+				Value: "cn-northwest-1",
+				Help:  "China (Ningxia) Region\nNeeds location constraint cn-northwest-1.",
+			}, {
+				Value: "us-gov-east-1",
+				Help:  "AWS GovCloud (US-East) Region\nNeeds location constraint us-gov-east-1.",
+			}, {
+				Value: "us-gov-west-1",
+				Help:  "AWS GovCloud (US) Region\nNeeds location constraint us-gov-west-1.",
 			}},
 		}, {
 			Name:     "region",
@@ -585,12 +615,12 @@ func init() {
 			}, {
 				Value: "us-east-2",
 				Help:  "US East (Ohio) Region.",
-			}, {
-				Value: "us-west-2",
-				Help:  "US West (Oregon) Region.",
 			}, {
 				Value: "us-west-1",
 				Help:  "US West (Northern California) Region.",
+			}, {
+				Value: "us-west-2",
+				Help:  "US West (Oregon) Region.",
 			}, {
 				Value: "ca-central-1",
 				Help:  "Canada (Central) Region.",
@@ -600,9 +630,15 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region.",
+			}, {
+				Value: "eu-west-3",
+				Help:  "EU (Paris) Region.",
 			}, {
 				Value: "eu-north-1",
 				Help:  "EU (Stockholm) Region.",
+			}, {
+				Value: "eu-south-1",
+				Help:  "EU (Milan) Region.",
 			}, {
 				Value: "EU",
 				Help:  "EU Region.",
@@ -617,16 +653,37 @@ func init() {
 				Help:  "Asia Pacific (Tokyo) Region.",
 			}, {
 				Value: "ap-northeast-2",
-				Help:  "Asia Pacific (Seoul)",
+				Help:  "Asia Pacific (Seoul) Region.",
+			}, {
+				Value: "ap-northeast-3",
+				Help:  "Asia Pacific (Osaka-Local) Region.",
 			}, {
 				Value: "ap-south-1",
-				Help:  "Asia Pacific (Mumbai)",
+				Help:  "Asia Pacific (Mumbai) Region.",
 			}, {
 				Value: "ap-east-1",
-				Help:  "Asia Pacific (Hong Kong)",
+				Help:  "Asia Pacific (Hong Kong) Region.",
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region.",
+			}, {
+				Value: "me-south-1",
+				Help:  "Middle East (Bahrain) Region.",
+			}, {
+				Value: "af-south-1",
+				Help:  "Africa (Cape Town) Region.",
+			}, {
+				Value: "cn-north-1",
+				Help:  "China (Beijing) Region",
+			}, {
+				Value: "cn-northwest-1",
+				Help:  "China (Ningxia) Region.",
+			}, {
+				Value: "us-gov-east-1",
+				Help:  "AWS GovCloud (US-East) Region.",
+			}, {
+				Value: "us-gov-west-1",
+				Help:  "AWS GovCloud (US) Region.",
 			}},
 		}, {
 			Name:     "location_constraint",
diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index 093ca0276..6a0ed2262 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -32,6 +32,7 @@ import (
 	"github.com/rclone/rclone/lib/readers"
 	sshagent "github.com/xanzy/ssh-agent"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
 )
 
 const (
@@ -86,6 +87,16 @@ in the new OpenSSH format can't be used.`,
 			Help: `Optional path to public key file.
 
 Set this if you have a signed certificate you want to use for authentication.` + env.ShellExpandHelp,
+		}, {
+			Name: "known_hosts_file",
+			Help: `Optional path to known_hosts file.
+
+Set this value to enable server host key validation.` + env.ShellExpandHelp,
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "~/.ssh/known_hosts",
+				Help:  "Use OpenSSH's known_hosts file",
+			}},
 		}, {
 			Name: "key_use_agent",
 			Help: `When set forces the usage of the ssh-agent.
@@ -195,6 +206,7 @@ type Options struct {
 	KeyFile           string `config:"key_file"`
 	KeyFilePass       string `config:"key_file_pass"`
 	PubKeyFile        string `config:"pubkey_file"`
+	KnownHostsFile    string `config:"known_hosts_file"`
 	KeyUseAgent       bool   `config:"key_use_agent"`
 	UseInsecureCipher bool   `config:"use_insecure_cipher"`
 	DisableHashCheck  bool   `config:"disable_hashcheck"`
@@ -414,6 +426,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if opt.Port == "" {
 		opt.Port = "22"
 	}
+
 	sshConfig := &ssh.ClientConfig{
 		User:            opt.User,
 		Auth:            []ssh.AuthMethod{},
@@ -422,6 +435,14 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		ClientVersion:   "SSH-2.0-" + fs.Config.UserAgent,
 	}
 
+	if opt.KnownHostsFile != "" {
+		hostcallback, err := knownhosts.New(opt.KnownHostsFile)
+		if err != nil {
+			return nil, errors.Wrap(err, "couldn't parse known_hosts_file")
+		}
+		sshConfig.HostKeyCallback = hostcallback
+	}
+
 	if opt.UseInsecureCipher {
 		sshConfig.Config.SetDefaults()
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc", "aes192-cbc", "aes256-cbc", "3des-cbc")
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 8b06b096a..ca01ecf1d 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -21,6 +21,7 @@ import (
 	"sync"
 	"time"
 
+	systemd "github.com/iguanesolutions/go-systemd/v5"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
@@ -35,6 +36,7 @@ import (
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/fs/rc/rcserver"
 	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/terminal"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -288,6 +290,11 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	}
 	fs.Debugf(nil, "%d go routines active\n", runtime.NumGoroutine())
 
+	if fs.Config.Progress && fs.Config.ProgressTerminalTitle {
+		// Clear terminal title
+		terminal.WriteTerminalTitle("")
+	}
+
 	// dump all running go-routines
 	if fs.Config.Dump&fs.DumpGoRoutines != 0 {
 		err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
@@ -364,6 +371,12 @@ func StartStats() func() {
 
 // initConfig is run by cobra after initialising the flags
 func initConfig() {
+	// Activate logger systemd support if systemd invocation ID is detected
+	_, sysdLaunch := systemd.GetInvocationID()
+	if sysdLaunch {
+		fs.Config.LogSystemdSupport = true // used during fslog.InitLogging()
+	}
+
 	// Start the logger
 	fslog.InitLogging()
 
@@ -379,6 +392,13 @@ func initConfig() {
 	// Write the args for debug purposes
 	fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
 
+	// Inform user about systemd log support now that we have a logger
+	if sysdLaunch {
+		fs.Debugf("rclone", "systemd logging support automatically activated")
+	} else if fs.Config.LogSystemdSupport {
+		fs.Debugf("rclone", "systemd logging support manually activated")
+	}
+
 	// Start the remote control server if configured
 	_, err = rcserver.Start(&rcflags.Opt)
 	if err != nil {
diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go
index 98de51f25..4a3e9e31c 100644
--- a/cmd/mountlib/mount.go
+++ b/cmd/mountlib/mount.go
@@ -11,7 +11,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/okzk/sdnotify"
+	sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs"
@@ -448,13 +448,13 @@ func Mount(VFS *vfs.VFS, mountpoint string, mount MountFn, opt *Options) error {
 
 	// Unmount on exit
 	fnHandle := atexit.Register(func() {
+		_ = sysdnotify.Stopping()
 		_ = unmount()
-		_ = sdnotify.Stopping()
 	})
 	defer atexit.Unregister(fnHandle)
 
 	// Notify systemd
-	if err := sdnotify.Ready(); err != nil && err != sdnotify.ErrSdNotifyNoSocket {
+	if err := sysdnotify.Ready(); err != nil {
 		return errors.Wrap(err, "failed to notify systemd")
 	}
 
@@ -479,8 +479,8 @@ waitloop:
 		}
 	}
 
+	_ = sysdnotify.Stopping()
 	_ = unmount()
-	_ = sdnotify.Stopping()
 
 	if err != nil {
 		return errors.Wrap(err, "failed to umount FUSE fs")
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 5705b4746..6526d1b92 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -413,3 +413,6 @@ put them back in again.` >}}
   * Muffin King <jinxz_k@live.com>
   * Christopher Stewart <6573710+1f47a@users.noreply.github.com>
   * Russell Cattelan <cattelan@digitalelves.com>
+  * gyutw <30371241+gyutw@users.noreply.github.com>
+  * Hekmon <edouardhur@gmail.com>
+  * LaSombra <lasombra@users.noreply.github.com>
diff --git a/docs/content/box.md b/docs/content/box.md
index fd1c6bc0b..1198f8bcd 100644
--- a/docs/content/box.md
+++ b/docs/content/box.md
@@ -404,6 +404,7 @@ Note that Box is case insensitive so you can't have a file called
 "Hello.doc" and one called "hello.doc".
 
 Box file names can't have the `\` character in.  rclone maps this to
-and from an identical looking unicode equivalent `＼`.
+and from an identical looking unicode equivalent `＼` (U+FF3C Fullwidth
+Reverse Solidus).
 
 Box only supports filenames up to 255 characters in length.
diff --git a/docs/content/crypt.md b/docs/content/crypt.md
index dbaa69f99..b5910e7f5 100644
--- a/docs/content/crypt.md
+++ b/docs/content/crypt.md
@@ -6,23 +6,26 @@ description: "Encryption overlay remote"
 {{< icon "fa fa-lock" >}}Crypt
 ----------------------------------------
 
-The `crypt` remote encrypts and decrypts another remote.
+Rclone `crypt` remotes encrypt and decrypt other remotes.
 
-To use it first set up the underlying remote following the config
-instructions for that remote.  You can also use a local pathname
-instead of a remote which will encrypt and decrypt from that directory
-which might be useful for encrypting onto a USB stick for example.
+To use `crypt`, first set up the underlying remote. Follow the `rclone
+config` instructions for that remote.
 
-First check your chosen remote is working - we'll call it
-`remote:path` in these docs.  Note that anything inside `remote:path`
-will be encrypted and anything outside won't.  This means that if you
-are using a bucket based remote (eg S3, B2, swift) then you should
-probably put the bucket in the remote `s3:bucket`. If you just use
-`s3:` then rclone will make encrypted bucket names too (if using file
-name encryption) which may or may not be what you want.
+`crypt` applied to a local pathname instead of a remote will
+encrypt and decrypt that directory, and can be used to encrypt USB
+removable drives.
 
-Now configure `crypt` using `rclone config`. We will call this one
-`secret` to differentiate it from the `remote`.
+Before configuring the crypt remote, check the underlying remote is
+working. In this example the underlying remote is called `remote:path`.
+Anything inside `remote:path` will be encrypted and anything outside
+will not. In the case of an S3 based underlying remote (eg Amazon S3,
+B2, Swift) it is generally advisable to define a crypt remote in the
+underlying remote `s3:bucket`. If `s3:` alone is specified alongside
+file name encryption, rclone will encrypt the bucket name.
+
+Configure `crypt` using `rclone config`. In this example the `crypt`
+remote is called `secret`, to differentiate it from the underlying
+`remote`.
 
 ```
 No remotes found - make a new one
@@ -96,49 +99,42 @@ d) Delete this remote
 y/e/d> y
 ```
 
-**Important** The password is stored in the config file is lightly
-obscured so it isn't immediately obvious what it is.  It is in no way
-secure unless you use config file encryption.
+**Important** The crypt password stored in `rclone.conf` is lightly
+obscured. That only protects it from cursory inspection. It is not
+secure unless encryption of `rclone.conf` is specified.
 
-A long passphrase is recommended, or you can use a random one.
+A long passphrase is recommended, or `rclone config` can generate a
+random one.
 
-The obscured password is created by using AES-CTR with a static key, with
-the salt stored verbatim at the beginning of the obscured password. This
-static key is shared by between all versions of rclone.
+The obscured password is created using AES-CTR with a static key. The
+salt is stored verbatim at the beginning of the obscured password. This
+static key is shared between all versions of rclone.
 
 If you reconfigure rclone with the same passwords/passphrases
 elsewhere it will be compatible, but the obscured version will be different
 due to the different salt.
 
-Note that rclone does not encrypt
+Rclone does not encrypt
 
   * file length - this can be calculated within 16 bytes
   * modification time - used for syncing
 
 ## Specifying the remote ##
 
-In normal use, make sure the remote has a `:` in. If you specify the
-remote without a `:` then rclone will use a local directory of that
-name.  So if you use a remote of `/path/to/secret/files` then rclone
-will encrypt stuff to that directory.  If you use a remote of `name`
-then rclone will put files in a directory called `name` in the current
-directory.
-
-If you specify the remote as `remote:path/to/dir` then rclone will
-store encrypted files in `path/to/dir` on the remote. If you are using
-file name encryption, then when you save files to
-`secret:subdir/subfile` this will store them in the unencrypted path
-`path/to/dir` but the `subdir/subpath` bit will be encrypted.
-
-Note that unless you want encrypted bucket names (which are difficult
-to manage because you won't know what directory they represent in web
-interfaces etc), you should probably specify a bucket, eg
-`remote:secretbucket` when using bucket based remotes such as S3,
-Swift, Hubic, B2, GCS.
+In normal use, ensure the remote has a `:` in. If specified without,
+rclone uses a local directory of that name.  For example if a remote
+`/path/to/secret/files` is specified, rclone encrypts content to that
+directory.  If a remote `name` is specified, rclone targets a directory
+`name` in the current directory.
+
+If remote `remote:path/to/dir` is specified, rclone stores encrypted
+files in `path/to/dir` on the remote. With file name encryption, files
+saved to `secret:subdir/subfile` are stored in the unencrypted path
+`path/to/dir` but the `subdir/subpath` element is encrypted.
 
 ## Example ##
 
-To test I made a little directory of files using "standard" file name
+Create the following file structure using "standard" file name
 encryption.
 
 ```
@@ -152,7 +148,7 @@ plaintext/
         └── file4.txt
 ```
 
-Copy these to the remote and list them back
+Copy these to the remote, and list them
 
 ```
 $ rclone -q copy plaintext secret:
@@ -164,7 +160,7 @@ $ rclone -q ls secret:
         9 subdir/file3.txt
 ```
 
-Now see what that looked like when encrypted
+The crypt remote looks like
 
 ```
 $ rclone -q ls remote:path
@@ -175,7 +171,7 @@ $ rclone -q ls remote:path
        56 86vhrsv86mpbtd3a0akjuqslj8/8njh1sk437gttmep3p70g81aps
 ```
 
-Note that this retains the directory structure which means you can do this
+The directory structure is preserved
 
 ```
 $ rclone -q ls secret:subdir
@@ -184,9 +180,9 @@ $ rclone -q ls secret:subdir
        10 subsubdir/file4.txt
 ```
 
-If don't use file name encryption then the remote will look like this
-- note the `.bin` extensions added to prevent the cloud provider
-attempting to interpret the data.
+Without file name encryption `.bin` extensions are added to underlying
+names. This prevents the cloud provider attempting to interpret file
+content.
 
 ```
 $ rclone -q ls remote:path
@@ -199,8 +195,6 @@ $ rclone -q ls remote:path
 
 ### File name encryption modes ###
 
-Here are some of the features of the file name encryption modes
-
 Off
 
   * doesn't hide file names or directory structure
@@ -219,17 +213,19 @@ Standard
 Obfuscation
 
 This is a simple "rotate" of the filename, with each file having a rot
-distance based on the filename. We store the distance at the beginning
-of the filename. So a file called "hello" may become "53.jgnnq".
+distance based on the filename. Rclone stores the distance at the
+beginning of the filename. A file called "hello" may become "53.jgnnq".
 
-This is not a strong encryption of filenames, but it may stop automated
-scanning tools from picking up on filename patterns. As such it's an
-intermediate between "off" and "standard". The advantage is that it
-allows for longer path segment names.
+Obfuscation is not a strong encryption of filenames, but hinders
+automated scanning tools picking up on filename patterns. It is an
+intermediate between "off" and "standard" which allows for longer path
+segment names.
 
 There is a possibility with some unicode based filenames that the
 obfuscation is weak and may map lower case characters to upper case
-equivalents.  You can not rely on this for strong protection.
+equivalents. 
+
+Obfuscation cannot be relied upon for strong protection.
 
   * file names very lightly obfuscated
   * file names can be longer than standard encryption
@@ -237,13 +233,14 @@ equivalents.  You can not rely on this for strong protection.
   * directory structure visible
   * identical files names will have identical uploaded names
 
-Cloud storage systems have various limits on file name length and
-total path length which you are more likely to hit using "Standard"
-file name encryption.  If you keep your file names to below 156
-characters in length then you should be OK on all providers.
+Cloud storage systems have limits on file name length and
+total path length which rclone is more likely to breach using
+"Standard" file name encryption.  Where file names are less thn 156
+characters in length issues should not be encountered, irrespective of
+cloud storage provider.
 
-There may be an even more secure file name encryption mode in the
-future which will address the long file name problem.
+An alternative, future rclone file name encryption mode may tolerate
+backend provider path length limits.
 
 ### Directory name encryption ###
 Crypt offers the option of encrypting dir names or leaving them intact.
@@ -269,10 +266,10 @@ Example:
 Crypt stores modification times using the underlying remote so support
 depends on that.
 
-Hashes are not stored for crypt.  However the data integrity is
+Hashes are not stored for crypt. However the data integrity is
 protected by an extremely strong crypto authenticator.
 
-Note that you should use the `rclone cryptcheck` command to check the
+Use the `rclone cryptcheck` command to check the
 integrity of a crypted remote instead of `rclone check` which can't
 check the checksums properly.
 
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 7248ccee1..f08ecb64d 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -1108,6 +1108,11 @@ Note: On Windows until [this bug](https://github.com/Azure/go-ansiterm/issues/26
 is fixed all non-ASCII characters will be replaced with `.` when
 `--progress` is in use.
 
+### --progress-terminal-title ###
+
+This flag, when used with `-P/--progress`, will print the string `ETA: %s`
+to the terminal title.
+
 ### -q, --quiet ###
 
 This flag will limit rclone's output to error messages only.
@@ -1253,11 +1258,17 @@ or with `--backup-dir`. See `--backup-dir` for more info.
 
 For example
 
-    rclone sync -i /path/to/local/file remote:current --suffix .bak
+    rclone copy -i /path/to/local/file remote:current --suffix .bak
 
-will sync `/path/to/local` to `remote:current`, but for any files
+will copy `/path/to/local` to `remote:current`, but for any files
 which would have been updated or deleted have .bak added.
 
+If using `rclone sync` with `--suffix` and without `--backup-dir` then
+it is recommended to put a filter rule in excluding the suffix
+otherwise the `sync` will delete the backup files.
+
+    rclone sync -i /path/to/local/file remote:current --suffix .bak --exclude "*.bak"
+
 ### --suffix-keep-extension ###
 
 When using `--suffix`, setting this causes rclone put the SUFFIX
diff --git a/docs/content/sftp.md b/docs/content/sftp.md
index c0c8cc1eb..aef29698e 100644
--- a/docs/content/sftp.md
+++ b/docs/content/sftp.md
@@ -148,6 +148,57 @@ Note: the cert must come first in the file.  e.g.
 cat id_rsa-cert.pub id_rsa > merged_key
 ```
 
+### Host key validation ###
+
+By default rclone will not check the server's host key for validation.  This
+can allow an attacker to replace a server with their own and if you use
+password authentication then this can lead to that password being exposed.
+
+Host key matching, using standard `known_hosts` files can be turned on by
+enabling the `known_hosts_file` option.  This can point to the file maintained
+by `OpenSSH` or can point to a unique file.
+
+e.g.
+
+```
+[remote]
+type = sftp
+host = example.com
+user = sftpuser
+pass = 
+known_hosts_file = ~/.ssh/known_hosts
+````
+
+There are some limitations:
+
+* `rclone` will not _manage_ this file for you.  If the key is missing or
+wrong then the connection will be refused.
+* If the server is set up for a certificate host key then the entry in
+the `known_hosts` file _must_ be the `@cert-authority` entry for the CA
+* Unlike `OpenSSH`, the libraries used by `rclone` do not permit (at time
+of writing) multiple host keys to be listed for a server.  Only the first
+entry is used.
+
+If the host key provided by the server does not match the one in the
+file (or is missing) then the connection will be aborted and an error
+returned such as
+
+    NewFs: couldn't connect SSH: ssh: handshake failed: knownhosts: key mismatch
+
+or
+
+    NewFs: couldn't connect SSH: ssh: handshake failed: knownhosts: key is unknown
+
+If you see an error such as
+
+    NewFs: couldn't connect SSH: ssh: handshake failed: ssh: no authorities for hostname: example.com:22
+
+then it is likely the server has presented a CA signed host certificate
+and you will need to add the appropriate `@cert-authority` entry.
+
+The `known_hosts_file` setting can be set during `rclone config` as an
+advanced option.
+
 ### ssh-agent on macOS ###
 
 Note that there seem to be various problems with using an ssh-agent on
@@ -320,6 +371,23 @@ Leave blank or set to false to enable hashing (recommended), set to true to disa
 
 Here are the advanced options specific to sftp (SSH/SFTP Connection).
 
+#### --sftp-known-hosts-file
+
+Optional path to known_hosts file.
+
+Set this value to enable server host key validation.
+
+Leading `~` will be expanded in the file name as will environment variables such as `${RCLONE_CONFIG_DIR}`.
+
+
+- Config:      known_hosts_file
+- Env Var:     RCLONE_SFTP_KNOWN_HOSTS_FILE
+- Type:        string
+- Default:     ""
+- Examples:
+    - "~/.ssh/known_hosts"
+        - Use OpenSSH's known_hosts file
+
 #### --sftp-ask-password
 
 Allow asking for SFTP password when needed.
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index fbea0114f..0c5ee2903 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -11,6 +11,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/lib/terminal"
 )
 
 // MaxCompletedTransfers specifies maximum number of completed transfers in startedTransfers list
@@ -282,6 +283,11 @@ func (s *StatsInfo) String() string {
 		xfrchkString,
 	)
 
+	if fs.Config.ProgressTerminalTitle {
+		// Writes ETA to the terminal title
+		terminal.WriteTerminalTitle("ETA: " + etaString(currentSize, totalSize, speed))
+	}
+
 	if !fs.Config.StatsOneLine {
 		_, _ = buf.WriteRune('\n')
 		errorDetails := ""
diff --git a/fs/config.go b/fs/config.go
index 3c04af7de..7b10dce13 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -42,6 +42,7 @@ var (
 type ConfigInfo struct {
 	LogLevel               LogLevel
 	StatsLogLevel          LogLevel
+	LogSystemdSupport      bool
 	UseJSONLog             bool
 	DryRun                 bool
 	Interactive            bool
@@ -106,6 +107,7 @@ type ConfigInfo struct {
 	StatsOneLineDateFormat string // If we want to customize the prefix
 	ErrorOnNoTransfer      bool   // Set appropriate exit code if no files transferred
 	Progress               bool
+	ProgressTerminalTitle  bool
 	Cookie                 bool
 	UseMmap                bool
 	CaCert                 string // Client Side CA
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index 275e64d05..1933ab905 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -110,6 +110,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.StringVarP(flagSet, &fs.Config.StatsOneLineDateFormat, "stats-one-line-date-format", "", fs.Config.StatsOneLineDateFormat, "Enables --stats-one-line-date and uses custom formatted date. Enclose date string in double quotes (\"). See https://golang.org/pkg/time/#Time.Format")
 	flags.BoolVarP(flagSet, &fs.Config.ErrorOnNoTransfer, "error-on-no-transfer", "", fs.Config.ErrorOnNoTransfer, "Sets exit code 9 if no files are transferred, useful in scripts")
 	flags.BoolVarP(flagSet, &fs.Config.Progress, "progress", "P", fs.Config.Progress, "Show progress during transfer.")
+	flags.BoolVarP(flagSet, &fs.Config.ProgressTerminalTitle, "progress-terminal-title", "", fs.Config.ProgressTerminalTitle, "Show progress on the terminal title. Requires -P/--progress.")
 	flags.BoolVarP(flagSet, &fs.Config.Cookie, "use-cookies", "", fs.Config.Cookie, "Enable session cookiejar.")
 	flags.BoolVarP(flagSet, &fs.Config.UseMmap, "use-mmap", "", fs.Config.UseMmap, "Use mmap allocator (see docs).")
 	flags.StringVarP(flagSet, &fs.Config.CaCert, "ca-cert", "", fs.Config.CaCert, "CA certificate used to verify servers")
@@ -123,6 +124,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions")
 	flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions")
 	flags.BoolVarP(flagSet, &fs.Config.RefreshTimes, "refresh-times", "", fs.Config.RefreshTimes, "Refresh the modtime of remote files.")
+	flags.BoolVarP(flagSet, &fs.Config.LogSystemdSupport, "log-systemd", "", fs.Config.LogSystemdSupport, "Activate systemd integration for the logger.")
 }
 
 // ParseHeaders converts the strings passed in via the header flags into HTTPOptions
diff --git a/fs/log.go b/fs/log.go
index 3baf00400..4bba8a922 100644
--- a/fs/log.go
+++ b/fs/log.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 
+	sysdjournald "github.com/iguanesolutions/go-systemd/v5/journald"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -70,7 +71,28 @@ func (l *LogLevel) Type() string {
 
 // LogPrint sends the text to the logger of level
 var LogPrint = func(level LogLevel, text string) {
-	text = fmt.Sprintf("%-6s: %s", level, text)
+	var prefix string
+	if Config.LogSystemdSupport {
+		switch level {
+		case LogLevelDebug:
+			prefix = sysdjournald.DebugPrefix
+		case LogLevelInfo:
+			prefix = sysdjournald.InfoPrefix
+		case LogLevelNotice:
+			prefix = sysdjournald.NoticePrefix
+		case LogLevelWarning:
+			prefix = sysdjournald.WarningPrefix
+		case LogLevelError:
+			prefix = sysdjournald.ErrPrefix
+		case LogLevelCritical:
+			prefix = sysdjournald.CritPrefix
+		case LogLevelAlert:
+			prefix = sysdjournald.AlertPrefix
+		case LogLevelEmergency:
+			prefix = sysdjournald.EmergPrefix
+		}
+	}
+	text = fmt.Sprintf("%s%-6s: %s", prefix, level, text)
 	_ = log.Output(4, text)
 }
 
diff --git a/fs/log/log.go b/fs/log/log.go
index 48c0bd45a..f505b99c3 100644
--- a/fs/log/log.go
+++ b/fs/log/log.go
@@ -90,14 +90,19 @@ func Stack(o interface{}, info string) {
 func InitLogging() {
 	flagsStr := "," + Opt.Format + ","
 	var flags int
-	if strings.Contains(flagsStr, ",date,") {
-		flags |= log.Ldate
-	}
-	if strings.Contains(flagsStr, ",time,") {
-		flags |= log.Ltime
-	}
-	if strings.Contains(flagsStr, ",microseconds,") {
-		flags |= log.Lmicroseconds
+	if !fs.Config.LogSystemdSupport {
+		if strings.Contains(flagsStr, ",date,") {
+			flags |= log.Ldate
+		}
+		if strings.Contains(flagsStr, ",time,") {
+			flags |= log.Ltime
+		}
+		if strings.Contains(flagsStr, ",microseconds,") {
+			flags |= log.Lmicroseconds
+		}
+		if strings.Contains(flagsStr, ",UTC,") {
+			flags |= log.LUTC
+		}
 	}
 	if strings.Contains(flagsStr, ",longfile,") {
 		flags |= log.Llongfile
@@ -105,9 +110,6 @@ func InitLogging() {
 	if strings.Contains(flagsStr, ",shortfile,") {
 		flags |= log.Lshortfile
 	}
-	if strings.Contains(flagsStr, ",UTC,") {
-		flags |= log.LUTC
-	}
 	log.SetFlags(flags)
 
 	// Log file output
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index b899e755d..56e75d47c 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -181,9 +181,12 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
 			return false
 		}
 		if ht == hash.None {
-			checksumWarning.Do(func() {
-				fs.Logf(dst.Fs(), "--checksum is in use but the source and destination have no hashes in common; falling back to --size-only")
-			})
+			common := src.Fs().Hashes().Overlap(dst.Fs().Hashes())
+			if common.Count() == 0 {
+				checksumWarning.Do(func() {
+					fs.Logf(dst.Fs(), "--checksum is in use but the source and destination have no hashes in common; falling back to --size-only")
+				})
+			}
 			fs.Debugf(src, "Size of src and dst objects identical")
 		} else {
 			fs.Debugf(src, "Size and %v of src and dst objects identical", ht)
@@ -1533,12 +1536,11 @@ func BackupDir(fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err
 				}
 			}
 		}
-	} else {
-		if srcFileName == "" {
-			return nil, fserrors.FatalError(errors.New("--suffix must be used with a file or with --backup-dir"))
-		}
+	} else if fs.Config.Suffix != "" {
 		// --backup-dir is not set but --suffix is - use the destination as the backupDir
 		backupDir = fdst
+	} else {
+		return nil, fserrors.FatalError(errors.New("internal error: BackupDir called when --backup-dir and --suffix both empty"))
 	}
 	if !CanServerSideMove(backupDir) {
 		return nil, fserrors.FatalError(errors.New("can't use --backup-dir on a remote which doesn't support server side move or copy"))
diff --git a/go.mod b/go.mod
index a8a257675..21e95089d 100644
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ require (
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/hanwen/go-fuse/v2 v2.0.3
+	github.com/iguanesolutions/go-systemd/v5 v5.0.0
 	github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e
 	github.com/jzelinskie/whirlpool v0.0.0-20170603002051-c19460b8caa6
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
@@ -35,7 +36,6 @@ require (
 	github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2
 	github.com/ncw/swift v1.0.52
 	github.com/nsf/termbox-go v0.0.0-20200418040025-38ba6e5628f1
-	github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.11.0
diff --git a/go.sum b/go.sum
index 5319ab629..4b5ec2d63 100644
--- a/go.sum
+++ b/go.sum
@@ -232,6 +232,8 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/iguanesolutions/go-systemd/v5 v5.0.0 h1:E4OUiBdmlD1IsClS6cmRIdzWBW8T8UBitCqYem7A1KY=
+github.com/iguanesolutions/go-systemd/v5 v5.0.0/go.mod h1:VPlzL6z0rXd3HU7oLkMoEqTWBhHClInYX9rP2U/+giI=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -314,8 +316,6 @@ github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S
 github.com/nsf/termbox-go v0.0.0-20200418040025-38ba6e5628f1 h1:lh3PyZvY+B9nFliSGTn5uFuqQQJGuNrD0MLCokv09ag=
 github.com/nsf/termbox-go v0.0.0-20200418040025-38ba6e5628f1/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd h1:+iAPaTbi1gZpcpDwe/BW1fx7Xoesv69hLNGPheoyhBs=
-github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd/go.mod h1:4soZNh0zW0LtYGdQ416i0jO0EIqMGcbtaspRS4BDvRQ=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
diff --git a/lib/terminal/terminal.go b/lib/terminal/terminal.go
index d29931380..b3007a93b 100644
--- a/lib/terminal/terminal.go
+++ b/lib/terminal/terminal.go
@@ -60,6 +60,9 @@ const (
 	HiMagentaBg = "\x1b[105m"
 	HiCyanBg    = "\x1b[106m"
 	HiWhiteBg   = "\x1b[107m"
+
+	ChangeTitle = "\033]0;"
+	BEL         = "\007"
 )
 
 var (
diff --git a/lib/terminal/terminal_normal.go b/lib/terminal/terminal_normal.go
index 787054eed..d8531ebf5 100644
--- a/lib/terminal/terminal_normal.go
+++ b/lib/terminal/terminal_normal.go
@@ -3,6 +3,7 @@
 package terminal
 
 import (
+	"fmt"
 	"os"
 
 	"golang.org/x/crypto/ssh/terminal"
@@ -29,3 +30,8 @@ func IsTerminal(fd int) bool {
 func ReadPassword(fd int) ([]byte, error) {
 	return terminal.ReadPassword(fd)
 }
+
+// WriteTerminalTitle writes a string to the terminal title
+func WriteTerminalTitle(title string) {
+	fmt.Printf(ChangeTitle + title + BEL)
+}
diff --git a/lib/terminal/terminal_unsupported.go b/lib/terminal/terminal_unsupported.go
index bd70d3409..d26fb6291 100644
--- a/lib/terminal/terminal_unsupported.go
+++ b/lib/terminal/terminal_unsupported.go
@@ -21,3 +21,8 @@ func IsTerminal(fd int) bool {
 func ReadPassword(fd int) ([]byte, error) {
 	return nil, errors.New("can't read password")
 }
+
+// WriteTerminalTitle writes a string to the terminal title
+func WriteTerminalTitle(title string) {
+	// Since there's nothing to return, this is a NOOP
+}
diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go
index 11958138b..eb181429a 100644
--- a/vfs/vfscache/cache.go
+++ b/vfs/vfscache/cache.go
@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	fscache "github.com/rclone/rclone/fs/cache"
@@ -466,9 +467,15 @@ func (c *Cache) retryFailedResets() {
 	if len(c.errItems) != 0 {
 		fs.Debugf(nil, "vfs cache reset: before redoing reset errItems = %v", c.errItems)
 		for itemName := range c.errItems {
-			_, _, err := c.item[itemName].Reset()
-			if err == nil || !fserrors.IsErrNoSpace(err) {
-				// TODO: not trying to handle non-ENOSPC errors yet
+			if retryItem, ok := c.item[itemName]; ok {
+				_, _, err := retryItem.Reset()
+				if err == nil || !fserrors.IsErrNoSpace(err) {
+					// TODO: not trying to handle non-ENOSPC errors yet
+					delete(c.errItems, itemName)
+				}
+			} else {
+				// The retry item was deleted because it was closed.
+				// No need to redo the failed reset now.
 				delete(c.errItems, itemName)
 			}
 		}
@@ -603,7 +610,7 @@ func (c *Cache) clean(removeCleanFiles bool) {
 	if os.IsNotExist(err) {
 		return
 	}
-
+	c.updateUsed()
 	c.mu.Lock()
 	oldItems, oldUsed := len(c.item), fs.SizeSuffix(c.used)
 	c.mu.Unlock()
@@ -652,7 +659,12 @@ func (c *Cache) clean(removeCleanFiles bool) {
 	c.mu.Unlock()
 	uploadsInProgress, uploadsQueued := c.writeback.Stats()
 
-	fs.Infof(nil, "vfs cache: cleaned: objects %d (was %d) in use %d, to upload %d, uploading %d, total size %v (was %v)", newItems, oldItems, totalInUse, uploadsQueued, uploadsInProgress, newUsed, oldUsed)
+	stats := fmt.Sprintf("objects %d (was %d) in use %d, to upload %d, uploading %d, total size %v (was %v)",
+		newItems, oldItems, totalInUse, uploadsQueued, uploadsInProgress, newUsed, oldUsed)
+	fs.Infof(nil, "vfs cache: cleaned: %s", stats)
+	if err = sysdnotify.Status(fmt.Sprintf("[%s] vfs cache: %s", time.Now().Format("15:04"), stats)); err != nil {
+		fs.Errorf(nil, "vfs cache: updating systemd status with current stats failed: %s", err)
+	}
 }
 
 // cleaner calls clean at regular intervals and upon being kicked for out-of-space condition
diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.go
index b7cde1d9a..6ed57f6d1 100644
--- a/vfs/vfscache/downloaders/downloaders.go
+++ b/vfs/vfscache/downloaders/downloaders.go
@@ -230,7 +230,11 @@ func (dls *Downloaders) Close(inErr error) (err error) {
 		}
 	}
 	dls.cancel()
+	// dls may have entered the periodical (every 5 seconds) kickWaiters() call
+	// unlock the mutex to allow it to finish so that we can get its dls.wg.Done()
+	dls.mu.Unlock()
 	dls.wg.Wait()
+	dls.mu.Lock()
 	dls.dls = nil
 	dls._dispatchWaiters()
 	dls._closeWaiters(inErr)
diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go
index 5a997f076..22ca1801d 100644
--- a/vfs/vfscache/item.go
+++ b/vfs/vfscache/item.go
@@ -43,6 +43,13 @@ import (
 // be taken before Item.mu. writeback may call into Item but Item may
 // **not** call writeback methods with Item.mu held
 
+// LL Item reset is invoked by cache cleaner for synchronous recovery
+// from ENOSPC errors. The reset operation removes the cache file and
+// closes/reopens the downloaders.  Although most parts of reset and
+// other item operations are done with the item mutex held, the mutex
+// is released during fd.WriteAt and downloaders calls. We use preAccess
+// and postAccess calls to serialize reset and other item operations.
+
 // Item is stored in the item map
 //
 // The Info field is written to the backing store to store status
@@ -310,6 +317,8 @@ func (item *Item) _truncateToCurrentSize() (err error) {
 // extended and the extended data will be filled with zeros. The
 // object will be marked as dirty in this case also.
 func (item *Item) Truncate(size int64) (err error) {
+	item.preAccess()
+	defer item.postAccess()
 	item.mu.Lock()
 	defer item.mu.Unlock()
 
@@ -429,6 +438,8 @@ func (item *Item) _dirty() {
 
 // Dirty marks the item as changed and needing writeback
 func (item *Item) Dirty() {
+	item.preAccess()
+	defer item.postAccess()
 	item.mu.Lock()
 	item._dirty()
 	item.mu.Unlock()
@@ -612,6 +623,8 @@ func (item *Item) store(ctx context.Context, storeFn StoreFn) (err error) {
 // Close the cache file
 func (item *Item) Close(storeFn StoreFn) (err error) {
 	// defer log.Trace(item.o, "Item.Close")("err=%v", &err)
+	item.preAccess()
+	defer item.postAccess()
 	var (
 		downloaders   *downloaders.Downloaders
 		syncWriteBack = item.c.opt.WriteBack <= 0
@@ -1169,6 +1182,7 @@ func (item *Item) setModTime(modTime time.Time) {
 // ReadAt bytes from the file at off
 func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
 	n = 0
+	var expBackOff int
 	for retries := 0; retries < fs.Config.LowLevelRetries; retries++ {
 		item.preAccess()
 		n, err = item.readAt(b, off)
@@ -1182,6 +1196,12 @@ func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
 			break
 		}
 		item.c.KickCleaner()
+		expBackOff = 2 << uint(retries)
+		time.Sleep(time.Duration(expBackOff) * time.Millisecond) // Exponential back-off the retries
+	}
+
+	if fserrors.IsErrNoSpace(err) {
+		fs.Errorf(item.name, "vfs cache: failed to _ensure cache after retries %v", err)
 	}
 
 	return n, err
@@ -1213,6 +1233,8 @@ func (item *Item) readAt(b []byte, off int64) (n int, err error) {
 
 // WriteAt bytes to the file at off
 func (item *Item) WriteAt(b []byte, off int64) (n int, err error) {
+	item.preAccess()
+	defer item.postAccess()
 	item.mu.Lock()
 	if item.fd == nil {
 		item.mu.Unlock()
@@ -1303,6 +1325,8 @@ func (item *Item) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped int, e
 // this means flushing the file system's in-memory copy of recently written
 // data to disk.
 func (item *Item) Sync() (err error) {
+	item.preAccess()
+	defer item.postAccess()
 	item.mu.Lock()
 	defer item.mu.Unlock()
 	if item.fd == nil {
@@ -1322,6 +1346,8 @@ func (item *Item) Sync() (err error) {
 
 // rename the item
 func (item *Item) rename(name string, newName string, newObj fs.Object) (err error) {
+	item.preAccess()
+	defer item.postAccess()
 	item.mu.Lock()
 
 	// stop downloader
@@ -1351,6 +1377,5 @@ func (item *Item) rename(name string, newName string, newObj fs.Object) (err err
 		_ = downloaders.Close(nil)
 	}
 	item.c.writeback.Rename(id, newName)
-
 	return err
 }


Test patch:
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index c29459f41..f25762608 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -1590,7 +1590,7 @@ func TestSyncCopyDest(t *testing.T) {
 }
 
 // Test with BackupDir set
-func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
+func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeepExtension bool) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 
@@ -1599,7 +1599,23 @@ func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
 	}
 	r.Mkdir(context.Background(), r.Fremote)
 
-	fs.Config.BackupDir = r.FremoteName + "/backup"
+	if backupDir != "" {
+		fs.Config.BackupDir = r.FremoteName + "/" + backupDir
+		backupDir += "/"
+	} else {
+		fs.Config.BackupDir = ""
+		backupDir = "dst/"
+		// Exclude the suffix from the sync otherwise the sync
+		// deletes the old backup files
+		flt, err := filter.NewFilter(nil)
+		require.NoError(t, err)
+		require.NoError(t, flt.AddRule("- *"+suffix))
+		oldFlt := filter.Active
+		filter.Active = flt
+		defer func() {
+			filter.Active = oldFlt
+		}()
+	}
 	fs.Config.Suffix = suffix
 	fs.Config.SuffixKeepExtension = suffixKeepExtension
 	defer func() {
@@ -1627,14 +1643,14 @@ func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
 	require.NoError(t, err)
 
 	// one should be moved to the backup dir and the new one installed
-	file1.Path = "backup/one" + suffix
+	file1.Path = backupDir + "one" + suffix
 	file1a.Path = "dst/one"
 	// two should be unchanged
 	// three should be moved to the backup dir
 	if suffixKeepExtension {
-		file3.Path = "backup/three" + suffix + ".txt"
+		file3.Path = backupDir + "three" + suffix + ".txt"
 	} else {
-		file3.Path = "backup/three.txt" + suffix
+		file3.Path = backupDir + "three.txt" + suffix
 	}
 
 	fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a)
@@ -1652,22 +1668,29 @@ func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
 	require.NoError(t, err)
 
 	// one should be moved to the backup dir and the new one installed
-	file1a.Path = "backup/one" + suffix
+	file1a.Path = backupDir + "one" + suffix
 	file1b.Path = "dst/one"
 	// two should be unchanged
 	// three should be moved to the backup dir
 	if suffixKeepExtension {
-		file3a.Path = "backup/three" + suffix + ".txt"
+		file3a.Path = backupDir + "three" + suffix + ".txt"
 	} else {
-		file3a.Path = "backup/three.txt" + suffix
+		file3a.Path = backupDir + "three.txt" + suffix
 	}
 
 	fstest.CheckItems(t, r.Fremote, file1b, file2, file3a, file1a)
 }
-func TestSyncBackupDir(t *testing.T)           { testSyncBackupDir(t, "", false) }
-func TestSyncBackupDirWithSuffix(t *testing.T) { testSyncBackupDir(t, ".bak", false) }
+func TestSyncBackupDir(t *testing.T) {
+	testSyncBackupDir(t, "backup", "", false)
+}
+func TestSyncBackupDirWithSuffix(t *testing.T) {
+	testSyncBackupDir(t, "backup", ".bak", false)
+}
 func TestSyncBackupDirWithSuffixKeepExtension(t *testing.T) {
-	testSyncBackupDir(t, "-2019-01-01", true)
+	testSyncBackupDir(t, "backup", "-2019-01-01", true)
+}
+func TestSyncBackupDirSuffixOnly(t *testing.T) {
+	testSyncBackupDir(t, "", ".bak", false)
 }
 
 // Test with Suffix set