Repository: rclone/rclone
Pull request #4242 (2020-05-17 00:13:40)

Cache - Add Support for Server Side Copy When Using Temp Upload
#### What is the purpose of this change?

This change adds support for server side copying to the cache backend (when wrapping a backend that supports it) when the `tmp_upload_path` option is specified. Previously, this configuration would result in the following behaviour from e.g. `mv`:

```text
user@HOSTNAME:/data/data# mv a.txt b.txt
mv: cannot move 'a.txt' to 'b.txt': Input/output error
```

rclone itself would emit a message similar to the following:

```text
ERROR : data/a.txt: Dir.Rename error: Fs "Encrypted drive 'data:'" can't rename files (no server side Move or Copy)
```

This was caused by the `cache` backend "masking out" Copy support from the wrapped backend due to the `local` backend not supporting it.

#### Was the change discussed in an issue or in the forum before?

https://github.com/rclone/rclone/issues/3206

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)


DWSR: I'm not clear on whether or not this should have additional tests written for it. It's a bug that wasn't caught by the existing tests and normally I would say that that's something that warrants adding a test, but I'm fuzzy on exactly how the integration tests are run and it appears that the integration tests currently test this scenario.

Patch:
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 000000000..97c3c26ca
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Rclone Forum Community Support
+    url: https://forum.rclone.org/
+    about: Please ask and answer questions here.
diff --git a/backend/cache/cache.go b/backend/cache/cache.go
index 8b1263cad..98ebe79b0 100644
--- a/backend/cache/cache.go
+++ b/backend/cache/cache.go
@@ -520,9 +520,6 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	// override only those features that use a temp fs and it doesn't support them
 	//f.features.ChangeNotify = f.ChangeNotify
 	if f.opt.TempWritePath != "" {
-		if f.tempFs.Features().Copy == nil {
-			f.features.Copy = nil
-		}
 		if f.tempFs.Features().Move == nil {
 			f.features.Move = nil
 		}
@@ -1533,6 +1530,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Errorf(src, "source remote (%v) doesn't support Copy", src.Fs())
 		return nil, fs.ErrorCantCopy
 	}
+	if f.opt.TempWritePath != "" && src.Fs() == f.tempFs {
+		return nil, fs.ErrorCantCopy
+	}
 	// the source must be a cached object or we abort
 	srcObj, ok := src.(*Object)
 	if !ok {
diff --git a/backend/googlephotos/pattern.go b/backend/googlephotos/pattern.go
index e54b7c4ab..961bbe585 100644
--- a/backend/googlephotos/pattern.go
+++ b/backend/googlephotos/pattern.go
@@ -53,6 +53,7 @@ var patterns = dirPatterns{
 				fs.NewDir(prefix+"album", f.dirTime()),
 				fs.NewDir(prefix+"shared-album", f.dirTime()),
 				fs.NewDir(prefix+"upload", f.dirTime()),
+				fs.NewDir(prefix+"feature", f.dirTime()),
 			}, nil
 		},
 	},
@@ -190,6 +191,28 @@ var patterns = dirPatterns{
 		re:     `^shared-album/(.+?)/([^/]+)$`,
 		isFile: true,
 	},
+	{
+		re: `^feature$`,
+		toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
+			return fs.DirEntries{
+				fs.NewDir(prefix+"favorites", f.dirTime()),
+			}, nil
+		},
+	},
+	{
+		re: `^feature/favorites$`,
+		toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
+			filter := featureFilter(ctx, f, match)
+			if err != nil {
+				return nil, err
+			}
+			return f.listDir(ctx, prefix, filter)
+		},
+	},
+	{
+		re:     `^feature/favorites/([^/]+)$`,
+		isFile: true,
+	},
 }.mustCompile()
 
 // mustCompile compiles the regexps in the dirPatterns
@@ -290,6 +313,24 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
 	return sf, nil
 }
 
+// featureFilter creates a filter for the Feature enum
+//
+// The API only supports one feature, FAVORITES, so hardcode that feature
+//
+// https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter
+func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) {
+	sf = api.SearchFilter{
+		Filters: &api.Filters{
+			FeatureFilter: &api.FeatureFilter{
+				IncludedFeatures: []string{
+					"FAVORITES",
+				},
+			},
+		},
+	}
+	return sf
+}
+
 // Turns an albumPath into entries
 //
 // These can either be synthetic directory entries if the album path
diff --git a/backend/local/local.go b/backend/local/local.go
index bcd6d9b68..86de84fa4 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -118,6 +118,16 @@ Windows/macOS and case sensitive for everything else.  Use this flag
 to override the default choice.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_sparse",
+			Help: `Disable sparse files for multi-thread downloads
+
+On Windows platforms rclone will make sparse files when doing
+multi-thread downloads. This avoids long pauses on large files where
+the OS zeros the file. However sparse files may be undesirable as they
+cause disk fragmentation and can be slow to work with.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -139,6 +149,7 @@ type Options struct {
 	OneFileSystem     bool                 `config:"one_file_system"`
 	CaseSensitive     bool                 `config:"case_sensitive"`
 	CaseInsensitive   bool                 `config:"case_insensitive"`
+	NoSparse          bool                 `config:"no_sparse"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -1095,6 +1106,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return o.lstat()
 }
 
+var sparseWarning sync.Once
+
 // OpenWriterAt opens with a handle for random access writes
 //
 // Pass in the remote desired and the size if known.
@@ -1122,10 +1135,15 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 	if err != nil {
 		fs.Debugf(o, "Failed to pre-allocate: %v", err)
 	}
-	// Set the file to be a sparse file (important on Windows)
-	err = file.SetSparse(out)
-	if err != nil {
-		fs.Debugf(o, "Failed to set sparse: %v", err)
+	if !f.opt.NoSparse && file.SetSparseImplemented {
+		sparseWarning.Do(func() {
+			fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable")
+		})
+		// Set the file to be a sparse file (important on Windows)
+		err = file.SetSparse(out)
+		if err != nil {
+			fs.Debugf(o, "Failed to set sparse: %v", err)
+		}
 	}
 
 	return out, nil
diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go
index 037a0d42b..6e58346a9 100644
--- a/backend/qingstor/qingstor.go
+++ b/backend/qingstor/qingstor.go
@@ -56,7 +56,7 @@ func init() {
 			Help: "QingStor Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.",
 		}, {
 			Name: "endpoint",
-			Help: "Enter a endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
+			Help: "Enter an endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
 		}, {
 			Name: "zone",
 			Help: "Zone to connect to.\nDefault is \"pek3a\".",
@@ -206,7 +206,7 @@ func (o *Object) split() (bucket, bucketPath string) {
 // Split an URL into three parts: protocol host and port
 func qsParseEndpoint(endpoint string) (protocol, host, port string, err error) {
 	/*
-	  Pattern to match a endpoint,
+	  Pattern to match an endpoint,
 	  eg: "http(s)://qingstor.com:443" --> "http(s)", "qingstor.com", 443
 	      "http(s)//qingstor.com"      --> "http(s)", "qingstor.com", ""
 	      "qingstor.com"               --> "", "qingstor.com", ""
diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index 7e5be399c..660e2a02f 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -69,6 +69,9 @@ func init() {
 			Name:       "pass",
 			Help:       "SSH password, leave blank to use ssh-agent.",
 			IsPassword: true,
+		}, {
+			Name: "key_pem",
+			Help: "Raw PEM-encoded private key, If specified, will override key_file parameter.",
 		}, {
 			Name: "key_file",
 			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
@@ -172,6 +175,7 @@ type Options struct {
 	User              string `config:"user"`
 	Port              string `config:"port"`
 	Pass              string `config:"pass"`
+	KeyPem            string `config:"key_pem"`
 	KeyFile           string `config:"key_file"`
 	KeyFilePass       string `config:"key_file_pass"`
 	KeyUseAgent       bool   `config:"key_use_agent"`
@@ -390,6 +394,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)
+	//keyPem := env.ShellExpand(opt.KeyPem)
 	// Add ssh agent-auth if no password or file specified
 	if (opt.Pass == "" && keyFile == "" && !opt.AskPassword) || opt.KeyUseAgent {
 		sshAgentClient, _, err := sshagent.New()
@@ -427,10 +432,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// Load key file if specified
-	if keyFile != "" {
-		key, err := ioutil.ReadFile(keyFile)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to read private key file")
+	if keyFile != "" || opt.KeyPem != "" {
+		var key []byte
+		if opt.KeyPem == "" {
+			key, err = ioutil.ReadFile(keyFile)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to read private key file")
+			}
+		} else {
+			// wrap in quotes because the config is a coming as a literal without them.
+			opt.KeyPem, err = strconv.Unquote("\"" + opt.KeyPem + "\"")
+			if err != nil {
+				return nil, errors.Wrap(err, "pem key not formatted properly")
+			}
+			key = []byte(opt.KeyPem)
 		}
 		clearpass := ""
 		if opt.KeyFilePass != "" {
diff --git a/cmd/backend/backend.go b/cmd/backend/backend.go
index 2d32ff447..12c5a3f33 100644
--- a/cmd/backend/backend.go
+++ b/cmd/backend/backend.go
@@ -59,7 +59,7 @@ Note to run these commands on a running backend then see
 [backend/command](/rc/#backend/command) in the rc docs.
 `,
 	RunE: func(command *cobra.Command, args []string) error {
-		cmd.CheckArgs(2, 1E6, command, args)
+		cmd.CheckArgs(2, 1e6, command, args)
 		name, remote := args[0], args[1]
 		cmd.Run(false, false, command, func() error {
 			// show help if remote is a backend name
diff --git a/cmd/config/config.go b/cmd/config/config.go
index 379506455..64450a6ee 100644
--- a/cmd/config/config.go
+++ b/cmd/config/config.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/cmd"
@@ -70,7 +71,8 @@ var configShowCommand = &cobra.Command{
 		if len(args) == 0 {
 			config.ShowConfig()
 		} else {
-			config.ShowRemote(args[0])
+			name := strings.TrimRight(args[0], ":")
+			config.ShowRemote(name)
 		}
 	},
 }
diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go
index 19c3aeaed..1994b1935 100644
--- a/cmd/hashsum/hashsum.go
+++ b/cmd/hashsum/hashsum.go
@@ -25,7 +25,7 @@ func init() {
 
 var commandDefinition = &cobra.Command{
 	Use:   "hashsum <hash> remote:path",
-	Short: `Produces an hashsum file for all the objects in the path.`,
+	Short: `Produces a hashsum file for all the objects in the path.`,
 	Long: `
 Produces a hash file for all the objects in the path using the hash
 named.  The output is in the same format as the standard
diff --git a/cmd/move/move.go b/cmd/move/move.go
index 80f96c201..66dbf9a7b 100644
--- a/cmd/move/move.go
+++ b/cmd/move/move.go
@@ -33,7 +33,7 @@ the remote does not support a server side directory move operation.
 
 If no filters are in use and if possible this will server side move
 ` + "`source:path`" + ` into ` + "`dest:path`" + `. After this ` + "`source:path`" + ` will no
-longer longer exist.
+longer exist.
 
 Otherwise for each file in ` + "`source:path`" + ` selected by the filters (if
 any) this will move it into ` + "`dest:path`" + `.  If possible a server side
diff --git a/cmd/serve/httplib/httplib.go b/cmd/serve/httplib/httplib.go
index 1bda5b91d..4e19c14d2 100644
--- a/cmd/serve/httplib/httplib.go
+++ b/cmd/serve/httplib/httplib.go
@@ -103,7 +103,7 @@ https.  You will need to supply the --cert and --key flags.  If you
 wish to do client side certificate validation then you will need to
 supply --client-ca also.
 
---cert should be a either a PEM encoded certificate or a concatenation
+--cert should be either a PEM encoded certificate or a concatenation
 of that with the CA certificate.  --key should be the PEM encoded
 private key and --client-ca should be the PEM encoded client
 certificate authority certificate.
diff --git a/docs/content/alias.md b/docs/content/alias.md
index 83738a7a0..d47b32d90 100644
--- a/docs/content/alias.md
+++ b/docs/content/alias.md
@@ -15,7 +15,7 @@ eg `remote:directory/subdirectory` or `/directory/subdirectory`.
 During the initial setup with `rclone config` you will specify the target
 remote. The target remote can either be a local path or another remote.
 
-Subfolders can be used in target remote. Assume a alias remote named `backup`
+Subfolders can be used in target remote. Assume an alias remote named `backup`
 with the target `mydrive:private/backup`. Invoking `rclone mkdir backup:desktop`
 is exactly the same as invoking `rclone mkdir mydrive:private/backup/desktop`.
 
@@ -25,7 +25,7 @@ Invoking `rclone mkdir backup:../desktop` is exactly the same as invoking
 The empty path is not allowed as a remote. To alias the current directory
 use `.` instead.
 
-Here is an example of how to make a alias called `remote` for local folder.
+Here is an example of how to make an alias called `remote` for local folder.
 First run:
 
      rclone config
diff --git a/docs/content/amazonclouddrive.md b/docs/content/amazonclouddrive.md
index 77fbb1fc4..9d69390fa 100644
--- a/docs/content/amazonclouddrive.md
+++ b/docs/content/amazonclouddrive.md
@@ -38,7 +38,7 @@ which pass through it.
 
 Since rclone doesn't currently have its own Amazon Drive credentials
 so you will either need to have your own `client_id` and
-`client_secret` with Amazon Drive, or use a a third party ouath proxy
+`client_secret` with Amazon Drive, or use a third party oauth proxy
 in which case you will need to enter `client_id`, `client_secret`,
 `auth_url` and `token_url`.
 
diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md
index 5f995cd8d..b13a2c4b4 100644
--- a/docs/content/azureblob.md
+++ b/docs/content/azureblob.md
@@ -301,7 +301,7 @@ MD5 sums are only uploaded with chunked files if the source has an MD5
 sum.  This will always be the case for a local to azure copy.
 
 ### Azure Storage Emulator Support ###
-You can test rlcone with storage emulator locally, to do this make sure azure storage emulator
+You can test rclone with storage emulator locally, to do this make sure azure storage emulator
 installed locally and set up a new remote with `rclone config` follow instructions described in
 introduction, set `use_emulator` config as `true`, you do not need to provide default account name
 or key if using emulator.
diff --git a/docs/content/cache.md b/docs/content/cache.md
index e3cfa0c8d..448b787bc 100644
--- a/docs/content/cache.md
+++ b/docs/content/cache.md
@@ -185,10 +185,10 @@ Affected settings:
 ##### Certificate Validation #####
 
 When the Plex server is configured to only accept secure connections, it is
-possible to use `.plex.direct` URL's to ensure certificate validation succeeds.
-These URL's are used by Plex internally to connect to the Plex server securely.
+possible to use `.plex.direct` URLs to ensure certificate validation succeeds.
+These URLs are used by Plex internally to connect to the Plex server securely.
 
-The format for this URL's is the following:
+The format for these URLs is the following:
 
 https://ip-with-dots-replaced.server-hash.plex.direct:32400/
 
@@ -241,7 +241,7 @@ there is a valid concern that the expiring cache listings can lead to cloud prov
 throttles or bans due to repeated queries on it for very large mounts.
 
 Some recommendations:
-- don't use a very small interval for entry informations (`--cache-info-age`)
+- don't use a very small interval for entry information (`--cache-info-age`)
 - while writes aren't yet optimised, you can still write through `cache` which gives you the advantage
 of adding the file in the cache at the same time if configured to do so.
 
@@ -405,7 +405,7 @@ The plex token for authentication - auto set normally
 
 #### --cache-plex-insecure
 
-Skip all certificate verifications when connecting to the Plex server
+Skip all certificate verification when connecting to the Plex server
 
 - Config:      plex_insecure
 - Env Var:     RCLONE_CACHE_PLEX_INSECURE
diff --git a/docs/content/changelog.md b/docs/content/changelog.md
index 642e0ec95..85c03504b 100644
--- a/docs/content/changelog.md
+++ b/docs/content/changelog.md
@@ -189,7 +189,7 @@ date: "2020-02-01"
     * copyurl
         * Add `--auto-filename` flag for using file name from URL in destination path (Denis)
     * serve dlna:
-        * Many compatability improvements (Dan Walters)
+        * Many compatibility improvements (Dan Walters)
         * Support for external srt subtitles (Dan Walters)
     * rc
         * Added command core/quit (Saksham Khanna)
@@ -220,7 +220,7 @@ date: "2020-02-01"
     * filter: Prevent mixing options when `--files-from` is in use (Michele Caci)
     * serve sftp: Fix crash on unsupported operations (eg Readlink) (Nick Craig-Wood)
 * Mount
-    * Allow files of unkown size to be read properly (Nick Craig-Wood)
+    * Allow files of unknown size to be read properly (Nick Craig-Wood)
     * Skip tests on <= 2 CPUs to avoid lockup (Nick Craig-Wood)
     * Fix panic on File.Open (Nick Craig-Wood)
     * Fix "mount_fusefs: -o timeout=: option not supported" on FreeBSD (Nick Craig-Wood)
@@ -259,7 +259,7 @@ date: "2020-02-01"
     * Fix signature v2_auth headers (Anthony Rusdi)
     * Fix encoding for control characters (Nick Craig-Wood)
     * Only ask for URL encoded directory listings if we need them on Ceph (Nick Craig-Wood)
-    * Add option for multipart failiure behaviour (Aleksandar Jankovic)
+    * Add option for multipart failure behaviour (Aleksandar Jankovic)
     * Support for multipart copy (庄天翼)
     * Fix nil pointer reference if no metadata returned for object (Nick Craig-Wood)
 * SFTP
@@ -695,7 +695,7 @@ date: "2020-02-01"
 * Jottacloud
     * Resume and deduplication support (Oliver Heyme)
     * Use token auth for all API requests Don't store password anymore (Sebastian Bünger)
-    * Add support for 2-factor authentification (Sebastian Bünger)
+    * Add support for 2-factor authentication (Sebastian Bünger)
 * Mega
     * Implement v2 account login which fixes logins for newer Mega accounts (Nick Craig-Wood)
     * Return error if an unknown length file is attempted to be uploaded (Nick Craig-Wood)
@@ -710,7 +710,7 @@ date: "2020-02-01"
     * Default `--qingstor-upload-concurrency` to 1 to work around bug (Nick Craig-Wood)
 * S3
     * Implement `--s3-upload-cutoff` for single part uploads below this (Nick Craig-Wood)
-    * Change `--s3-upload-concurrency` default to 4 to increase perfomance (Nick Craig-Wood)
+    * Change `--s3-upload-concurrency` default to 4 to increase performance (Nick Craig-Wood)
     * Add `--s3-bucket-acl` to control bucket ACL (Nick Craig-Wood)
     * Auto detect region for buckets on operation failure (Nick Craig-Wood)
     * Add GLACIER storage class (William Cocker)
@@ -892,7 +892,7 @@ date: "2020-02-01"
     * Add link sharing support (jackyzy823)
 * S3
     * Use custom pacer, to retry operations when reasonable (Craig Miskell)
-    * Use configured server-side-encryption and storace class options when calling CopyObject() (Paul Kohout)
+    * Use configured server-side-encryption and storage class options when calling CopyObject() (Paul Kohout)
     * Make `--s3-v2-auth` flag (Nick Craig-Wood)
     * Fix v2 auth on files with spaces (Nick Craig-Wood)
 * Union
@@ -1018,8 +1018,8 @@ Point release to fix hubic and azureblob backends.
     * Remove leading / from paths (Nick Craig-Wood)
 * Swift
     * Add `storage_policy` (Ruben Vandamme)
-    * Make it so just `storage_url` or `auth_token` can be overidden (Nick Craig-Wood)
-    * Fix server side copy bug for unusal file names (Nick Craig-Wood)
+    * Make it so just `storage_url` or `auth_token` can be overridden (Nick Craig-Wood)
+    * Fix server side copy bug for unusual file names (Nick Craig-Wood)
     * Remove leading / from paths (Nick Craig-Wood)
 * WebDAV
     * Ensure we call MKCOL with a URL with a trailing / for QNAP interop (Nick Craig-Wood)
@@ -1519,7 +1519,7 @@ Point release to fix hubic and azureblob backends.
         * --old-sync-method deprecated - the remaining uses are covered by --fast-list
         * This involved a major re-write of all the listing code
     * Add --tpslimit and --tpslimit-burst to limit transactions per second
-        * this is useful in conjuction with `rclone mount` to limit external apps
+        * this is useful in conjunction with `rclone mount` to limit external apps
     * Add --stats-log-level so can see --stats without -v
     * Print password prompts to stderr - Hraban Luyat
     * Warn about duplicate files when syncing
@@ -1853,7 +1853,7 @@ Point release to fix hubic and azureblob backends.
     * Add ap-northeast-2 (Seoul) and ap-south-1 (Mumbai) regions.
     * Skip setting the modified time for objects > 5GB as it isn't possible.
 * Backblaze B2
-    * Add --b2-versions flag so old versions can be listed and retreived.
+    * Add --b2-versions flag so old versions can be listed and retrieved.
     * Treat 403 errors (eg cap exceeded) as fatal.
     * Implement cleanup command for deleting old file versions.
     * Make error handling compliant with B2 integrations notes.
@@ -1979,7 +1979,7 @@ Point release to fix hubic and azureblob backends.
     * Add `--delete-before`, `--delete-during`, `--delete-after` flags.
     * Add `--memprofile` flag to debug memory use.
     * Warn the user about files with same name but different case
-    * Make `--include` rules add their implict exclude * at the end of the filter list
+    * Make `--include` rules add their implicit exclude * at the end of the filter list
     * Deprecate compiling with go1.3
 * Amazon Drive
     * Fix download of files > 10 GB
diff --git a/docs/content/chunker.md b/docs/content/chunker.md
index 80cb8e415..27dd19a8c 100644
--- a/docs/content/chunker.md
+++ b/docs/content/chunker.md
@@ -212,7 +212,7 @@ guarantee given hash for all files. If wrapped remote doesn't support it,
 chunker will then add metadata to all files, even small. However, this can
 double the amount of small files in storage and incur additional service charges.
 You can even use chunker to force md5/sha1 support in any other remote
-at expence of sidecar meta objects by setting eg. `chunk_type=sha1all`
+at expense of sidecar meta objects by setting eg. `chunk_type=sha1all`
 to force hashsums and `chunk_size=1P` to effectively disable chunking.
 
 Normally, when a file is copied to chunker controlled remote, chunker
@@ -293,7 +293,7 @@ Chunker will not automatically rename existing chunks when you run
 Beware that in result of this some files which have been treated as chunks
 before the change can pop up in directory listings as normal files
 and vice versa. The same warning holds for the chunk size.
-If you desperately need to change critical chunking setings, you should
+If you desperately need to change critical chunking settings, you should
 run data migration as described above.
 
 If wrapped remote is case insensitive, the chunker overlay will inherit
diff --git a/docs/content/crypt.md b/docs/content/crypt.md
index 41bec7ec7..64242cd75 100644
--- a/docs/content/crypt.md
+++ b/docs/content/crypt.md
@@ -113,7 +113,7 @@ due to the different salt.
 
 Note that rclone does not encrypt
 
-  * file length - this can be calcuated within 16 bytes
+  * file length - this can be calculated within 16 bytes
   * modification time - used for syncing
 
 ## Specifying the remote ##
@@ -367,7 +367,7 @@ names, or for debugging purposes.
 
 ## Backing up a crypted remote ##
 
-If you wish to backup a crypted remote, it it recommended that you use
+If you wish to backup a crypted remote, it is recommended that you use
 `rclone sync` on the encrypted files, and make sure the passwords are
 the same in the new encrypted remote.
 
@@ -451,7 +451,7 @@ files.
 File names are encrypted segment by segment - the path is broken up
 into `/` separated strings and these are encrypted individually.
 
-File segments are padded using using PKCS#7 to a multiple of 16 bytes
+File segments are padded using PKCS#7 to a multiple of 16 bytes
 before encryption.
 
 They are then encrypted with EME using AES with 256 bit key. EME
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 2d64733e3..8104fb9a5 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -522,7 +522,7 @@ to disable server side move and server side copy use:
 
     --disable move,copy
 
-The features can be put in in any case.
+The features can be put in any case.
 
 To see a list of which features can be disabled use:
 
@@ -794,13 +794,13 @@ Rclone will exit with exit code 8 if the transfer limit is reached.
 This modifies the behavior of `--max-transfer`
 Defaults to `--cutoff-mode=hard`.
 
-Specifiying `--cutoff-mode=hard` will stop transferring immediately
+Specifying `--cutoff-mode=hard` will stop transferring immediately
 when Rclone reaches the limit.
 
-Specifiying `--cutoff-mode=soft` will stop starting new transfers
+Specifying `--cutoff-mode=soft` will stop starting new transfers
 when Rclone reaches the limit.
 
-Specifiying `--cutoff-mode=cautious` will try to prevent Rclone
+Specifying `--cutoff-mode=cautious` will try to prevent Rclone
 from reaching the limit.
 
 ### --modify-window=TIME ###
@@ -827,7 +827,7 @@ time) then each thread writes directly into the file at the correct
 place.  This means that rclone won't create fragmented or sparse files
 and there won't be any assembly time at the end of the transfer.
 
-The number of threads used to dowload is controlled by
+The number of threads used to download is controlled by
 `--multi-thread-streams`.
 
 Use `-vv` if you wish to see info about the threads.
@@ -844,6 +844,12 @@ with any source.
 as they are faster without unless `--multi-thread-streams` is set
 explicitly.
 
+**NB** on Windows using multi-thread downloads will cause the
+resulting files to be [sparse](https://en.wikipedia.org/wiki/Sparse_file).
+Use `--local-no-sparse` to disable sparse files (which may cause long
+delays at the start of downloads) or disable multi-thread downloads
+with `--multi-thread-streams 0`
+
 ### --multi-thread-streams=N ###
 
 When using multi thread downloads (see above `--multi-thread-cutoff`)
@@ -1224,7 +1230,7 @@ Note also that `--track-renames` is incompatible with
 ### --track-renames-strategy (hash,modtime) ###
 
 This option changes the matching criteria for `--track-renames` to match
-by any combination of modtime, hash, size. Matchig by size is always enabled
+by any combination of modtime, hash, size. Matching by size is always enabled
 no matter what option is selected here. This also means
 that it enables `--track-renames` support for encrypted destinations.
 If nothing is specified, the default option is matching by hashes.
@@ -1371,7 +1377,7 @@ Prints the version number
 SSL/TLS options
 ---------------
 
-The outoing SSL/TLS connections rclone makes can be controlled with
+The outgoing SSL/TLS connections rclone makes can be controlled with
 these options.  For example this can be very useful with the HTTP or
 WebDAV backends. Rclone HTTP servers have their own set of
 configuration for SSL/TLS which you can find in their documentation.
diff --git a/docs/content/drive.md b/docs/content/drive.md
index 99cded3b8..b470204e3 100644
--- a/docs/content/drive.md
+++ b/docs/content/drive.md
@@ -678,7 +678,7 @@ videos.
 Setting this flag will cause Google photos and videos to return a
 blank MD5 checksum.
 
-Google photos are identifed by being in the "photos" space.
+Google photos are identified by being in the "photos" space.
 
 Corrupted checksums are caused by Google modifying the image/video but
 not updating the checksum.
@@ -877,7 +877,7 @@ Keep new head revision of each file forever.
 
 Show sizes as storage quota usage, not actual size.
 
-Show the size of a file as the the storage quota used. This is the
+Show the size of a file as the storage quota used. This is the
 current version plus any older versions that have been set to keep
 forever.
 
@@ -1028,7 +1028,7 @@ etc as rclone knows to ignore the size when doing the transfer.
 However an unfortunate consequence of this is that you may not be able
 to download Google docs using `rclone mount`. If it doesn't work you
 will get a 0 sized file.  If you try again the doc may gain its
-correct size and be downloadable. Whther it will work on not depends
+correct size and be downloadable. Whether it will work on not depends
 on the application accessing the mount and the OS you are running -
 experiment to find out if it does work for you!
 
@@ -1101,7 +1101,7 @@ of "External" above, but this has not been tested/documented so far).
 6.  Click on the "+ CREATE CREDENTIALS" button at the top of the screen,
 then select "OAuth client ID".
 
-7. Choose an application type of "other", and click "Create". (the
+7. Choose an application type of "Desktop app", and click "Create". (the
 default name is fine)
 
 8. It will show you a client ID and client secret.  Use these values
diff --git a/docs/content/fichier.md b/docs/content/fichier.md
index b5272831b..8717b20f5 100644
--- a/docs/content/fichier.md
+++ b/docs/content/fichier.md
@@ -7,7 +7,7 @@ date: "2015-10-14"
 <i class="fa fa-archive"></i> 1Fichier
 -----------------------------------------
 
-This is a backend for the [1ficher](https://1fichier.com) cloud
+This is a backend for the [1fichier](https://1fichier.com) cloud
 storage service. Note that a Premium subscription is required to use
 the API.
 
diff --git a/docs/content/filtering.md b/docs/content/filtering.md
index acab891eb..a97ca8648 100644
--- a/docs/content/filtering.md
+++ b/docs/content/filtering.md
@@ -308,7 +308,7 @@ This reads a list of file names from the file passed in and **only**
 these files are transferred.  The **filtering rules are ignored**
 completely if you use this option.
 
-`--files-from` expects a list of files as it's input. Leading / trailing
+`--files-from` expects a list of files as its input. Leading / trailing
 whitespace is stripped from the input lines and lines starting with `#`
 and `;` are ignored.
 
diff --git a/docs/content/flags.md b/docs/content/flags.md
index 647829fcc..bb090ad18 100755
--- a/docs/content/flags.md
+++ b/docs/content/flags.md
@@ -344,7 +344,7 @@ and may be set in the config file.
       --qingstor-chunk-size SizeSuffix               Chunk size to use for uploading. (default 4M)
       --qingstor-connection-retries int              Number of connection retries. (default 3)
       --qingstor-encoding MultiEncoder               This sets the encoding for the backend. (default Slash,Ctl,InvalidUtf8)
-      --qingstor-endpoint string                     Enter a endpoint URL to connection QingStor API.
+      --qingstor-endpoint string                     Enter an endpoint URL to connection QingStor API.
       --qingstor-env-auth                            Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank.
       --qingstor-secret-access-key string            QingStor Secret Access Key (password)
       --qingstor-upload-concurrency int              Concurrency for multipart uploads. (default 1)
diff --git a/docs/content/ftp.md b/docs/content/ftp.md
index 4dc738197..8da270ef0 100644
--- a/docs/content/ftp.md
+++ b/docs/content/ftp.md
@@ -132,7 +132,7 @@ Note that not all FTP servers can have all characters in file names, for example
 
 FTP supports implicit FTP over TLS servers (FTPS). This has to be enabled
 in the config for the remote. The default FTPS port is `990` so the
-port will likely have to be explictly set in the config for the remote.
+port will likely have to be explicitly set in the config for the remote.
 
 <!--- autogenerated options start - DO NOT EDIT, instead edit fs.RegInfo in backend/ftp/ftp.go then run make backenddocs -->
 ### Standard Options
diff --git a/docs/content/googlephotos.md b/docs/content/googlephotos.md
index 6a66d69a4..6ab42611a 100644
--- a/docs/content/googlephotos.md
+++ b/docs/content/googlephotos.md
@@ -173,10 +173,14 @@ into albums.
 - shared-album
     - album name
     - album name/sub
+- feature
+    - favorites
+        - file1.jpg
+        - file2.jpg
 ```
 
 There are two writable parts of the tree, the `upload` directory and
-sub directories of the the `album` directory.
+sub directories of the `album` directory.
 
 The `upload` directory is for uploading files you don't want to put
 into albums. This will be empty to start with and will contain the
diff --git a/docs/content/gui.md b/docs/content/gui.md
index 22b3e6aa7..ecaf3b306 100644
--- a/docs/content/gui.md
+++ b/docs/content/gui.md
@@ -73,7 +73,7 @@ The flag `--rc-web-gui` is shorthand for
 - `--rc-pass <random password>`
 - `--rc-serve`
 
-These flags can be overidden as desired.
+These flags can be overridden as desired.
 
 See also the [rclone rcd documentation](https://rclone.org/commands/rclone_rcd/).
 
@@ -95,14 +95,14 @@ If you want to run the GUI behind a proxy at `/rclone` you could use these flags
 - `--rc-baseurl rclone`
 - `--rc-htpasswd /path/to/htpasswd`
 
-Or instead of htpassword if you just want a single user and password:
+Or instead of htpasswd if you just want a single user and password:
 
 - `--rc-user me`
 - `--rc-pass mypassword`
 
 ## Project
 
-The GUI is being developed in the: [rclone/rclone-webui-react respository](https://github.com/rclone/rclone-webui-react).
+The GUI is being developed in the: [rclone/rclone-webui-react repository](https://github.com/rclone/rclone-webui-react).
 
 Bug reports and contributions are very welcome :-)
 
diff --git a/docs/content/install.sh b/docs/content/install.sh
index 90c400c93..7804627bc 100755
--- a/docs/content/install.sh
+++ b/docs/content/install.sh
@@ -9,7 +9,7 @@
 
 set -e
 
-#when adding a tool to the list make sure to also add it's corresponding command further in the script
+#when adding a tool to the list make sure to also add its corresponding command further in the script
 unzip_tools_list=('unzip' '7z' 'busybox')
 
 usage() { echo "Usage: curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md
index bed6cda28..acb278625 100644
--- a/docs/content/jottacloud.md
+++ b/docs/content/jottacloud.md
@@ -161,7 +161,7 @@ as they can't be used in XML strings.
 ### Deleting files
 
 By default rclone will send all files to the trash when deleting files. They will be permanently
-deleted automatically after 30 days. You may bypass the trash and permanently delete files immeditaly
+deleted automatically after 30 days. You may bypass the trash and permanently delete files immediately
 by using the [--jottacloud-hard-delete](#jottacloud-hard-delete) flag, or set the equivalent environment variable.
 Emptying the trash is supported by the [cleanup](/commands/rclone_cleanup/) command.
 
diff --git a/docs/content/local.md b/docs/content/local.md
index 1e62b5e1c..c27b899e4 100644
--- a/docs/content/local.md
+++ b/docs/content/local.md
@@ -428,7 +428,7 @@ See: the [encoding section in the overview](/overview/#encoding) for more info.
 
 Here are the commands specific to the local backend.
 
-Run them with with
+Run them with
 
     rclone backend COMMAND remote:
 
diff --git a/docs/content/mailru.md b/docs/content/mailru.md
index 1409f3cf9..054f00b0d 100644
--- a/docs/content/mailru.md
+++ b/docs/content/mailru.md
@@ -18,7 +18,7 @@ Currently it is recommended to disable 2FA on Mail.ru accounts intended for rclo
 - Deleted files are by default moved to the trash
 - Files and directories can be shared via public links
 - Partial uploads or streaming are not supported, file size must be known before upload
-- Maximum file size is limited to 2G for a free acount, unlimited for paid accounts
+- Maximum file size is limited to 2G for a free account, unlimited for paid accounts
 - Storage keeps hash for all files and performs transparent deduplication,
   the hash algorithm is a modified SHA1
 - If a particular file is already present in storage, one can quickly submit file hash
diff --git a/docs/content/mega.md b/docs/content/mega.md
index 3ddd03aa7..bb89acba0 100644
--- a/docs/content/mega.md
+++ b/docs/content/mega.md
@@ -109,7 +109,7 @@ Use `rclone dedupe` to fix duplicated files.
 
 Mega remotes seem to get blocked (reject logins) under "heavy use".
 We haven't worked out the exact blocking rules but it seems to be
-related to fast paced, sucessive rclone commands.
+related to fast paced, successive rclone commands.
 
 For example, executing this command 90 times in a row `rclone link
 remote:file` will cause the remote to become "blocked". This is not an
@@ -140,7 +140,7 @@ approach.
 
 Note that once blocked, the use of other tools (such as megacmd) is
 not a sure workaround: following megacmd login times have been
-observed in sucession for blocked remote: 7 minutes, 20 min, 30min, 30
+observed in succession for blocked remote: 7 minutes, 20 min, 30min, 30
 min, 30min. Web access looks unaffected though.
 
 Investigation is continuing in relation to workarounds based on
diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md
index c00be83fe..ee0886901 100644
--- a/docs/content/onedrive.md
+++ b/docs/content/onedrive.md
@@ -132,7 +132,7 @@ Client ID and Key by following the steps below:
 2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI` Enter `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use.
 3. Under `manage` select `Certificates & secrets`, click `New client secret`. Copy and keep that secret for later use.
 4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`.
-5. Search and select the follwing permssions: `Files.Read`, `Files.ReadWrite`, `Files.Read.All`, `Files.ReadWrite.All`, `offline_access`, `User.Read`. Once selected click `Add permissions` at the bottom.
+5. Search and select the following permissions: `Files.Read`, `Files.ReadWrite`, `Files.Read.All`, `Files.ReadWrite.All`, `offline_access`, `User.Read`. Once selected click `Add permissions` at the bottom.
 
 Now the application is complete. Run `rclone config` to create or edit a OneDrive remote.
 Supply the app ID and password as Client ID and Secret, respectively. rclone will walk you through the remaining steps.
diff --git a/docs/content/qingstor.md b/docs/content/qingstor.md
index d52769798..ba8b06c7c 100644
--- a/docs/content/qingstor.md
+++ b/docs/content/qingstor.md
@@ -43,7 +43,7 @@ QingStor Access Key ID - leave blank for anonymous access or runtime credentials
 access_key_id> access_key
 QingStor Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
 secret_access_key> secret_key
-Enter a endpoint URL to connection QingStor API.
+Enter an endpoint URL to connection QingStor API.
 Leave blank will use the default value "https://qingstor.com:443"
 endpoint>
 Zone connect to. Default is "pek3a".
@@ -182,7 +182,7 @@ Leave blank for anonymous access or runtime credentials.
 
 #### --qingstor-endpoint
 
-Enter a endpoint URL to connection QingStor API.
+Enter an endpoint URL to connection QingStor API.
 Leave blank will use the default value "https://qingstor.com:443"
 
 - Config:      endpoint
@@ -260,7 +260,7 @@ Concurrency for multipart uploads.
 This is the number of chunks of the same file that are uploaded
 concurrently.
 
-NB if you set this to > 1 then the checksums of multpart uploads
+NB if you set this to > 1 then the checksums of multipart uploads
 become corrupted (the uploads themselves are not corrupted though).
 
 If you are uploading small numbers of large file over high speed link
diff --git a/docs/content/rc.md b/docs/content/rc.md
index d698732e0..8352ac2bd 100644
--- a/docs/content/rc.md
+++ b/docs/content/rc.md
@@ -260,7 +260,7 @@ $ rclone rc job/list
 
 ### Assigning operations to groups with _group = <value>
 
-Each rc call has it's own stats group for tracking it's metrics. By default
+Each rc call has its own stats group for tracking its metrics. By default
 grouping is done by the composite group name from prefix `job/` and  id of the
 job like so `job/1`.
 
@@ -426,7 +426,7 @@ Eg
     }
 
 
-If the rate parameter is not suppied then the bandwidth is queried
+If the rate parameter is not supplied then the bandwidth is queried
 
     rclone rc core/bwlimit
     {
diff --git a/docs/content/s3.md b/docs/content/s3.md
index bbd8fe1cf..d17d00a1e 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -1320,7 +1320,7 @@ storage_class =
 
 [Spaces](https://www.digitalocean.com/products/object-storage/) is an [S3-interoperable](https://developers.digitalocean.com/documentation/spaces/) object storage service from cloud provider DigitalOcean.
 
-To connect to DigitalOcean Spaces you will need an access key and secret key. These can be retrieved on the "[Applications & API](https://cloud.digitalocean.com/settings/api/tokens)" page of the DigitalOcean control panel. They will be needed when promted by `rclone config` for your `access_key_id` and `secret_access_key`.
+To connect to DigitalOcean Spaces you will need an access key and secret key. These can be retrieved on the "[Applications & API](https://cloud.digitalocean.com/settings/api/tokens)" page of the DigitalOcean control panel. They will be needed when prompted by `rclone config` for your `access_key_id` and `secret_access_key`.
 
 When prompted for a `region` or `location_constraint`, press enter to use the default value. The region must be included in the `endpoint` setting (e.g. `nyc3.digitaloceanspaces.com`). The default values can be used for other settings.
 
diff --git a/docs/content/sftp.md b/docs/content/sftp.md
index bc8a17253..f99926a34 100644
--- a/docs/content/sftp.md
+++ b/docs/content/sftp.md
@@ -107,10 +107,20 @@ The SFTP remote supports three authentication methods:
 Key files should be PEM-encoded private key files. For instance `/home/$USER/.ssh/id_rsa`.
 Only unencrypted OpenSSH or PEM encrypted files are supported.
 
-If you don't specify `pass` or `key_file` then rclone will attempt to contact an ssh-agent.
+The key file can be specified in either an external file (key_file) or contained within the 
+rclone config file (key_pem).  If using key_pem in the config file, the entry should be on a
+single line with new line ('\n' or '\r\n') separating lines.  i.e. 
+
+key_pem = -----BEGIN RSA PRIVATE KEY-----\nMaMbaIXtE\n0gAMbMbaSsd\nMbaass\n-----END RSA PRIVATE KEY-----
+
+This will generate it correctly for key_pem for use in the config:  
+
+    awk '{printf "%s\\n", $0}' < ~/.ssh/id_rsa
+
+If you don't specify `pass`, `key_file`, or `key_pem` then rclone will attempt to contact an ssh-agent.
 
 You can also specify `key_use_agent` to force the usage of an ssh-agent. In this case
-`key_file` can also be specified to force the usage of a specific key in the ssh-agent.
+`key_file` or `key_pem` can also be specified to force the usage of a specific key in the ssh-agent.
 
 Using an ssh-agent is the only way to load encrypted OpenSSH keys at the moment.
 
diff --git a/docs/content/swift.md b/docs/content/swift.md
index 060afadcb..35541852a 100644
--- a/docs/content/swift.md
+++ b/docs/content/swift.md
@@ -480,7 +480,7 @@ The modified time is stored as metadata on the object as
 `X-Object-Meta-Mtime` as floating point since the epoch accurate to 1
 ns.
 
-This is a defacto standard (used in the official python-swiftclient
+This is a de facto standard (used in the official python-swiftclient
 amongst others) for storing the modification time for an object.
 
 ### Restricted filename characters
diff --git a/docs/content/union.md b/docs/content/union.md
index 47138c1ac..b66267498 100644
--- a/docs/content/union.md
+++ b/docs/content/union.md
@@ -60,7 +60,7 @@ Some policies rely on quota information. These policies should be used only if y
 | lus, eplus | Used           |
 | lno, eplno | Objects        |
 
-To check if your upstream support the field, run `rclone about remote: [flags]` and see if the reuqired field exists.
+To check if your upstream supports the field, run `rclone about remote: [flags]` and see if the required field exists.
 
 #### Filters
 
@@ -74,7 +74,7 @@ If all remotes are filtered an error will be returned.
 
 #### Policy descriptions
 
-THe policies definition are inspired by [trapexit/mergerfs](https://github.com/trapexit/mergerfs) but not exactly the same. Some policy definition could be different due to the much larger latency of remote file systems.
+The policies definition are inspired by [trapexit/mergerfs](https://github.com/trapexit/mergerfs) but not exactly the same. Some policy definition could be different due to the much larger latency of remote file systems.
 
 | Policy           | Description                                                |
 |------------------|------------------------------------------------------------|
diff --git a/docs/content/webdav.md b/docs/content/webdav.md
index 0ffa99d7e..6f23b8200 100644
--- a/docs/content/webdav.md
+++ b/docs/content/webdav.md
@@ -222,7 +222,7 @@ First, you need to get your remote's URL:
 - Now take a look at your address bar, the URL should look like this:
   `https://[YOUR-DOMAIN]-my.sharepoint.com/personal/[YOUR-EMAIL]/_layouts/15/onedrive.aspx`
 
-You'll only need this URL upto the email address. After that, you'll
+You'll only need this URL up to the email address. After that, you'll
 most likely want to add "/Documents". That subdirectory contains
 the actual data stored on your OneDrive.
 
diff --git a/fs/accounting/token_bucket.go b/fs/accounting/token_bucket.go
index 7671918c8..13058f68d 100644
--- a/fs/accounting/token_bucket.go
+++ b/fs/accounting/token_bucket.go
@@ -177,7 +177,7 @@ Eg
     }
 
 
-If the rate parameter is not suppied then the bandwidth is queried
+If the rate parameter is not supplied then the bandwidth is queried
 
     rclone rc core/bwlimit
     {
diff --git a/lib/file/preallocate_other.go b/lib/file/preallocate_other.go
index 5eae4ceab..87b01799e 100644
--- a/lib/file/preallocate_other.go
+++ b/lib/file/preallocate_other.go
@@ -4,11 +4,19 @@ package file
 
 import "os"
 
+// PreallocateImplemented is a constant indicating whether the
+// implementation of Preallocate actually does anything.
+const PreallocateImplemented = false
+
 // PreAllocate the file for performance reasons
 func PreAllocate(size int64, out *os.File) error {
 	return nil
 }
 
+// SetSparseImplemented is a constant indicating whether the
+// implementation of SetSparse actually does anything.
+const SetSparseImplemented = false
+
 // SetSparse makes the file be a sparse file
 func SetSparse(out *os.File) error {
 	return nil
diff --git a/lib/file/preallocate_unix.go b/lib/file/preallocate_unix.go
index 0bdd25ee5..8c3df5238 100644
--- a/lib/file/preallocate_unix.go
+++ b/lib/file/preallocate_unix.go
@@ -18,6 +18,10 @@ var (
 	fallocFlagsIndex int32
 )
 
+// PreallocateImplemented is a constant indicating whether the
+// implementation of Preallocate actually does anything.
+const PreallocateImplemented = true
+
 // PreAllocate the file for performance reasons
 func PreAllocate(size int64, out *os.File) error {
 	if size <= 0 {
@@ -45,6 +49,10 @@ again:
 	return err
 }
 
+// SetSparseImplemented is a constant indicating whether the
+// implementation of SetSparse actually does anything.
+const SetSparseImplemented = false
+
 // SetSparse makes the file be a sparse file
 func SetSparse(out *os.File) error {
 	return nil
diff --git a/lib/file/preallocate_windows.go b/lib/file/preallocate_windows.go
index f956536f7..f238c5c73 100644
--- a/lib/file/preallocate_windows.go
+++ b/lib/file/preallocate_windows.go
@@ -32,6 +32,10 @@ type ioStatusBlock struct {
 	Status, Information uintptr
 }
 
+// PreallocateImplemented is a constant indicating whether the
+// implementation of Preallocate actually does anything.
+const PreallocateImplemented = true
+
 // PreAllocate the file for performance reasons
 func PreAllocate(size int64, out *os.File) error {
 	if size <= 0 {
@@ -82,6 +86,10 @@ const (
 	FSCTL_SET_SPARSE = 0x000900c4
 )
 
+// SetSparseImplemented is a constant indicating whether the
+// implementation of SetSparse actually does anything.
+const SetSparseImplemented = true
+
 // SetSparse makes the file be a sparse file
 func SetSparse(out *os.File) error {
 	err := syscall.DeviceIoControl(syscall.Handle(out.Fd()), FSCTL_SET_SPARSE, nil, 0, nil, 0, nil, nil)
diff --git a/vfs/read.go b/vfs/read.go
index a89b1099b..c44d9118f 100644
--- a/vfs/read.go
+++ b/vfs/read.go
@@ -5,7 +5,6 @@ import (
 	"io"
 	"os"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/pkg/errors"
@@ -213,6 +212,43 @@ func (fh *ReadFileHandle) ReadAt(p []byte, off int64) (n int, err error) {
 	return fh.readAt(p, off)
 }
 
+// This waits for *poff to equal off or aborts after the timeout.
+//
+// Waits here potentially affect all seeks so need to keep them short
+//
+// Call with fh.mu Locked
+func waitSequential(what string, remote string, cond *sync.Cond, maxWait time.Duration, poff *int64, off int64) {
+	var (
+		timeout = time.NewTimer(maxWait)
+		done    = make(chan struct{})
+		abort   = false
+	)
+	go func() {
+		select {
+		case <-timeout.C:
+			// take the lock to make sure that cond.Wait() is called before
+			// cond.Broadcast. NB cond.L == mu
+			cond.L.Lock()
+			// set abort flag and give all the waiting goroutines a kick on timeout
+			abort = true
+			fs.Debugf(remote, "aborting in-sequence %s wait, off=%d", what, off)
+			cond.Broadcast()
+			cond.L.Unlock()
+		case <-done:
+		}
+	}()
+	for *poff != off && !abort {
+		fs.Debugf(remote, "waiting for in-sequence %s to %d for %v", what, off, maxWait)
+		cond.Wait()
+	}
+	// tidy up end timer
+	close(done)
+	timeout.Stop()
+	if *poff != off {
+		fs.Debugf(remote, "failed to wait for in-sequence %s to %d", what, off)
+	}
+}
+
 // Implementation of ReadAt - call with lock held
 func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
 	// defer log.Trace(fh.remote, "p[%d], off=%d", len(p), off)("n=%d, err=%v", &n, &err)
@@ -230,35 +266,7 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
 		maxBuf = len(p)
 	}
 	if gap := off - fh.offset; gap > 0 && gap < int64(8*maxBuf) {
-		// Set a background timer so we don't wait for long
-		// Waits here potentially affect all seeks so need to keep them short
-		// The default time here was made by finding the
-		// smallest when mounting a local backend that didn't
-		// cause seeks.
-		maxWait := fh.file.VFS().Opt.ReadWait
-		timeout := time.NewTimer(maxWait)
-		done := make(chan struct{})
-		abort := int32(0)
-		go func() {
-			select {
-			case <-timeout.C:
-				// set abort flag an give all the waiting goroutines a kick on timeout
-				atomic.StoreInt32(&abort, 1)
-				fs.Debugf(fh.remote, "aborting in-sequence read wait, off=%d", off)
-				fh.cond.Broadcast()
-			case <-done:
-			}
-		}()
-		for fh.offset != off && atomic.LoadInt32(&abort) == 0 {
-			fs.Debugf(fh.remote, "waiting for in-sequence read to %d for %v", off, maxWait)
-			fh.cond.Wait()
-		}
-		// tidy up end timer
-		close(done)
-		timeout.Stop()
-		if fh.offset != off {
-			fs.Debugf(fh.remote, "failed to wait for in-sequence read to %d", off)
-		}
+		waitSequential("read", fh.remote, fh.cond, fh.file.VFS().Opt.ReadWait, &fh.offset, off)
 	}
 	doSeek := off != fh.offset
 	if doSeek && fh.noSeek {
diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.go
index b22e69b2a..4ac2af361 100644
--- a/vfs/vfscommon/options.go
+++ b/vfs/vfscommon/options.go
@@ -53,5 +53,5 @@ var DefaultOpt = Options{
 	CacheMaxSize:      -1,
 	CaseInsensitive:   runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
 	WriteWait:         1000 * time.Millisecond,
-	ReadWait:          5 * time.Millisecond,
+	ReadWait:          20 * time.Millisecond,
 }
diff --git a/vfs/write.go b/vfs/write.go
index b7401bbe9..dd45fdcfd 100644
--- a/vfs/write.go
+++ b/vfs/write.go
@@ -5,7 +5,6 @@ import (
 	"io"
 	"os"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -131,28 +130,7 @@ func (fh *WriteFileHandle) writeAt(p []byte, off int64) (n int, err error) {
 		return 0, ECLOSED
 	}
 	if fh.offset != off {
-		// Set a background timer so we don't wait forever
-		maxWait := fh.file.VFS().Opt.WriteWait
-		timeout := time.NewTimer(maxWait)
-		done := make(chan struct{})
-		abort := int32(0)
-		go func() {
-			select {
-			case <-timeout.C:
-				// set abort flag an give all the waiting goroutines a kick on timeout
-				atomic.StoreInt32(&abort, 1)
-				fh.cond.Broadcast()
-			case <-done:
-			}
-		}()
-		// Wait for an in-sequence write or abort
-		for fh.offset != off && atomic.LoadInt32(&abort) == 0 {
-			// fs.Debugf(fh.remote, "waiting for in-sequence write to %d", off)
-			fh.cond.Wait()
-		}
-		// tidy up end timer
-		close(done)
-		timeout.Stop()
+		waitSequential("write", fh.remote, fh.cond, fh.file.VFS().Opt.WriteWait, &fh.offset, off)
 	}
 	if fh.offset != off {
 		fs.Errorf(fh.remote, "WriteFileHandle.Write: can't seek in file without --vfs-cache-mode >= writes")


Test patch:
diff --git a/backend/googlephotos/pattern_test.go b/backend/googlephotos/pattern_test.go
index f7aafdd3c..8c9d535c6 100644
--- a/backend/googlephotos/pattern_test.go
+++ b/backend/googlephotos/pattern_test.go
@@ -155,6 +155,38 @@ func TestPatternMatch(t *testing.T) {
 			wantPrefix:  "file.jpg/",
 			wantPattern: &patterns[5],
 		},
+		{
+			root:        "",
+			itemPath:    "feature",
+			isFile:      false,
+			wantMatch:   []string{"feature"},
+			wantPrefix:  "feature/",
+			wantPattern: &patterns[23],
+		},
+		{
+			root:        "feature/favorites",
+			itemPath:    "",
+			isFile:      false,
+			wantMatch:   []string{"feature/favorites"},
+			wantPrefix:  "",
+			wantPattern: &patterns[24],
+		},
+		{
+			root:        "feature",
+			itemPath:    "favorites",
+			isFile:      false,
+			wantMatch:   []string{"feature/favorites"},
+			wantPrefix:  "favorites/",
+			wantPattern: &patterns[24],
+		},
+		{
+			root:        "feature/favorites",
+			itemPath:    "file.jpg",
+			isFile:      true,
+			wantMatch:   []string{"feature/favorites/file.jpg", "file.jpg"},
+			wantPrefix:  "file.jpg/",
+			wantPattern: &patterns[25],
+		},
 	} {
 		t.Run(fmt.Sprintf("#%d,root=%q,itemPath=%q,isFile=%v", testNumber, test.root, test.itemPath, test.isFile), func(t *testing.T) {
 			gotMatch, gotPrefix, gotPattern := patterns.match(test.root, test.itemPath, test.isFile)