Repository: rclone/rclone
Pull request #4501 (2020-08-12 20:24:37)

docs: Add alternative workaround for issue #2036
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?

Adds alternative workaround to known issue

#### Was the change discussed in an issue or in the forum before?

Issue #2036 

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/.gitignore b/.gitignore
index d6da3db99..9a877432c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,4 @@ rclone.iml
 *.test
 *.log
 *.iml
+fuzz-build.zip
diff --git a/Makefile b/Makefile
index c150fadc3..832a61ed7 100644
--- a/Makefile
+++ b/Makefile
@@ -7,24 +7,26 @@ RELEASE_TAG := $(shell git tag -l --points-at HEAD)
 VERSION := $(shell cat VERSION)
 # Last tag on this branch
 LAST_TAG := $(shell git describe --tags --abbrev=0)
+# Next version
+NEXT_VERSION := $(shell echo $(VERSION) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
 # If we are working on a release, override branch to master
 ifdef RELEASE_TAG
 	BRANCH := master
+	LAST_TAG := $(shell git describe --abbrev=0 --tags $(VERSION)^)
 endif
-TAG_BRANCH := -$(BRANCH)
-BRANCH_PATH := branch/
+TAG_BRANCH := .$(BRANCH)
+BRANCH_PATH := branch/$(BRANCH)/
 # If building HEAD or master then unset TAG_BRANCH and BRANCH_PATH
 ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	TAG_BRANCH :=
 	BRANCH_PATH :=
 endif
-# Make version suffix -DDD-gCCCCCCCC (D=commits since last relase, C=Commit) or blank
-VERSION_SUFFIX := $(shell git describe --abbrev=8 --tags | perl -lpe 's/^v\d+\.\d+\.\d+//; s/^-(\d+)/"-".sprintf("%03d",$$1)/e;')
-# TAG is current version + number of commits since last release + branch
+# Make version suffix -beta.NNNN.CCCCCCCC (N=Commit number, C=Commit)
+VERSION_SUFFIX := -beta.$(shell git rev-list --count HEAD).$(shell git show --no-patch --no-notes --pretty='%h' HEAD)
+# TAG is current version + commit number + commit + branch
 TAG := $(VERSION)$(VERSION_SUFFIX)$(TAG_BRANCH)
-NEXT_VERSION := $(shell echo $(VERSION) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
-ifndef RELEASE_TAG
-	TAG := $(TAG)-beta
+ifdef RELEASE_TAG
+	TAG := $(RELEASE_TAG)
 endif
 GO_VERSION := $(shell go version)
 ifdef BETA_SUBDIR
@@ -56,7 +58,6 @@ vars:
 	@echo BRANCH="'$(BRANCH)'"
 	@echo TAG="'$(TAG)'"
 	@echo VERSION="'$(VERSION)'"
-	@echo NEXT_VERSION="'$(NEXT_VERSION)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
@@ -220,25 +221,24 @@ fetch_binaries:
 serve:	website
 	cd docs && hugo server -v -w --disableFastRender
 
-tag:	doc
-	@echo "Old tag is $(VERSION)"
-	@echo "New tag is $(NEXT_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)\"\n" | gofmt > fs/version.go
-	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_VERSION)" > VERSION
-	git tag -s -m "Version $(NEXT_VERSION)" $(NEXT_VERSION)
-	bin/make_changelog.py $(LAST_TAG) $(NEXT_VERSION) > docs/content/changelog.md.new
+tag:	retag doc
+	bin/make_changelog.py $(LAST_TAG) $(VERSION) > docs/content/changelog.md.new
 	mv docs/content/changelog.md.new docs/content/changelog.md
 	@echo "Edit the new changelog in docs/content/changelog.md"
 	@echo "Then commit all the changes"
-	@echo git commit -m \"Version $(NEXT_VERSION)\" -a -v
+	@echo git commit -m \"Version $(VERSION)\" -a -v
 	@echo "And finally run make retag before make cross etc"
 
 retag:
+	@echo "Version is $(VERSION)"
 	git tag -f -s -m "Version $(VERSION)" $(VERSION)
 
 startdev:
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(VERSION)-DEV\"\n" | gofmt > fs/version.go
+	@echo "Version is $(VERSION)"
+	@echo "Next version is $(NEXT_VERSION)"
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
+	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
+	echo "$(NEXT_VERSION)" > VERSION
 	git commit -m "Start $(VERSION)-DEV development" fs/version.go
 
 winzip:
diff --git a/RELEASE.md b/RELEASE.md
index 57bcef32f..eeb4d05aa 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -69,6 +69,8 @@ this will be done already.
 
 Now
 
+  * FIXME this is now broken with new semver layout - needs fixing
+  * FIXME the TAG=${NEW_TAG} shouldn't be necessary any more
   * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
   * Test (see above)
diff --git a/VERSION b/VERSION
index 5c77185a7..55274d266 100644
--- a/VERSION
+++ b/VERSION
@@ -1,1 +1,1 @@
-v1.52.3
+v1.53.0
diff --git a/backend/alias/alias.go b/backend/alias/alias.go
index f708ca265..873320e98 100644
--- a/backend/alias/alias.go
+++ b/backend/alias/alias.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
@@ -46,9 +47,5 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if strings.HasPrefix(opt.Remote, name+":") {
 		return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
 	}
-	fsInfo, configName, fsPath, config, err := fs.ConfigFs(opt.Remote)
-	if err != nil {
-		return nil, err
-	}
-	return fsInfo.NewFs(configName, fspath.JoinRootPath(fsPath, root), config)
+	return cache.Get(fspath.JoinRootPath(opt.Remote, root))
 }
diff --git a/backend/cache/cache.go b/backend/cache/cache.go
index 08fd83034..f91fb3e3d 100644
--- a/backend/cache/cache.go
+++ b/backend/cache/cache.go
@@ -361,15 +361,10 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, errors.Wrapf(err, "failed to clean root path %q", rootPath)
 	}
 
-	wInfo, wName, wPath, wConfig, err := fs.ConfigFs(opt.Remote)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", opt.Remote)
-	}
-
-	remotePath := fspath.JoinRootPath(wPath, rootPath)
-	wrappedFs, wrapErr := wInfo.NewFs(wName, remotePath, wConfig)
+	remotePath := fspath.JoinRootPath(opt.Remote, rootPath)
+	wrappedFs, wrapErr := cache.Get(remotePath)
 	if wrapErr != nil && wrapErr != fs.ErrorIsFile {
-		return nil, errors.Wrapf(wrapErr, "failed to make remote %s:%s to wrap", wName, remotePath)
+		return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath)
 	}
 	var fsErr error
 	fs.Debugf(name, "wrapped %v:%v at root %v", wrappedFs.Name(), wrappedFs.Root(), rpath)
@@ -390,6 +385,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 		cleanupChan:      make(chan bool, 1),
 		notifiedRemotes:  make(map[string]bool),
 	}
+	cache.PinUntilFinalized(f.Fs, f)
 	f.rateLimiter = rate.NewLimiter(rate.Limit(float64(opt.Rps)), opt.TotalWorkers)
 
 	f.plexConnector = &plexConnector{}
diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go
index 74bebacef..5bdfa7a4d 100644
--- a/backend/chunker/chunker.go
+++ b/backend/chunker/chunker.go
@@ -24,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
@@ -238,15 +239,18 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, errors.New("can't point remote at itself - check the value of the remote setting")
 	}
 
-	baseInfo, baseName, basePath, baseConfig, err := fs.ConfigFs(remote)
+	baseName, basePath, err := fspath.Parse(remote)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
 	}
+	if baseName != "" {
+		baseName += ":"
+	}
 	// Look for a file first
 	remotePath := fspath.JoinRootPath(basePath, rpath)
-	baseFs, err := baseInfo.NewFs(baseName, remotePath, baseConfig)
+	baseFs, err := cache.Get(baseName + remotePath)
 	if err != fs.ErrorIsFile && err != nil {
-		return nil, errors.Wrapf(err, "failed to make remote %s:%q to wrap", baseName, remotePath)
+		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", baseName+remotePath)
 	}
 	if !operations.CanServerSideMove(baseFs) {
 		return nil, errors.New("can't use chunker on a backend which doesn't support server side move or copy")
@@ -258,6 +262,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		root: rpath,
 		opt:  *opt,
 	}
+	cache.PinUntilFinalized(f.base, f)
 	f.dirSort = true // processEntries requires that meta Objects prerun data chunks atm.
 
 	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType); err != nil {
@@ -271,7 +276,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// (yet can't satisfy fstest.CheckListing, will ignore)
 	if err == nil && !f.useMeta && strings.Contains(rpath, "/") {
 		firstChunkPath := f.makeChunkName(remotePath, 0, "", "")
-		_, testErr := baseInfo.NewFs(baseName, firstChunkPath, baseConfig)
+		_, testErr := cache.Get(baseName + firstChunkPath)
 		if testErr == fs.ErrorIsFile {
 			err = testErr
 		}
diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go
index 831af4261..0ba57d9e4 100644
--- a/backend/crypt/crypt.go
+++ b/backend/crypt/crypt.go
@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
@@ -158,24 +159,25 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	if strings.HasPrefix(remote, name+":") {
 		return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting")
 	}
-	wInfo, wName, wPath, wConfig, err := fs.ConfigFs(remote)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
-	}
 	// Make sure to remove trailing . reffering to the current dir
 	if path.Base(rpath) == "." {
 		rpath = strings.TrimSuffix(rpath, ".")
 	}
 	// Look for a file first
-	remotePath := fspath.JoinRootPath(wPath, cipher.EncryptFileName(rpath))
-	wrappedFs, err := wInfo.NewFs(wName, remotePath, wConfig)
-	// if that didn't produce a file, look for a directory
-	if err != fs.ErrorIsFile {
-		remotePath = fspath.JoinRootPath(wPath, cipher.EncryptDirName(rpath))
-		wrappedFs, err = wInfo.NewFs(wName, remotePath, wConfig)
+	var wrappedFs fs.Fs
+	if rpath == "" {
+		wrappedFs, err = cache.Get(remote)
+	} else {
+		remotePath := fspath.JoinRootPath(remote, cipher.EncryptFileName(rpath))
+		wrappedFs, err = cache.Get(remotePath)
+		// if that didn't produce a file, look for a directory
+		if err != fs.ErrorIsFile {
+			remotePath = fspath.JoinRootPath(remote, cipher.EncryptDirName(rpath))
+			wrappedFs, err = cache.Get(remotePath)
+		}
 	}
 	if err != fs.ErrorIsFile && err != nil {
-		return nil, errors.Wrapf(err, "failed to make remote %s:%q to wrap", wName, remotePath)
+		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remote)
 	}
 	f := &Fs{
 		Fs:     wrappedFs,
@@ -184,6 +186,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		opt:    *opt,
 		cipher: cipher,
 	}
+	cache.PinUntilFinalized(f.Fs, f)
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
@@ -438,7 +441,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	if do == nil {
 		return fs.ErrorCantPurge
 	}
-	return do(ctx, dir)
+	return do(ctx, f.cipher.EncryptDirName(dir))
 }
 
 // Copy src to this remote using server side copy operations.
diff --git a/backend/drive/drive.go b/backend/drive/drive.go
index 02efb3f50..8e5f4b09f 100755
--- a/backend/drive/drive.go
+++ b/backend/drive/drive.go
@@ -281,6 +281,11 @@ commands (copy, sync, etc), and with all other commands too.`,
 			Default:  false,
 			Help:     "Only show files that are in the trash.\nThis will show trashed files in their original directory structure.",
 			Advanced: true,
+		}, {
+			Name:     "starred_only",
+			Default:  false,
+			Help:     "Only show files that are starred.",
+			Advanced: true,
 		}, {
 			Name:     "formats",
 			Default:  "",
@@ -513,6 +518,7 @@ type Options struct {
 	SkipChecksumGphotos       bool                 `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool                 `config:"shared_with_me"`
 	TrashedOnly               bool                 `config:"trashed_only"`
+	StarredOnly               bool                 `config:"starred_only"`
 	Extensions                string               `config:"formats"`
 	ExportExtensions          string               `config:"export_formats"`
 	ImportExtensions          string               `config:"import_formats"`
@@ -696,6 +702,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		}
 		query = append(query, q)
 	}
+
 	// Search with sharedWithMe will always return things listed in "Shared With Me" (without any parents)
 	// We must not filter with parent when we try list "ROOT" with drive-shared-with-me
 	// If we need to list file inside those shared folders, we must search it without sharedWithMe
@@ -707,8 +714,16 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		if parentsQuery.Len() > 1 {
 			_, _ = parentsQuery.WriteString(" or ")
 		}
-		if f.opt.SharedWithMe && dirID == f.rootFolderID {
-			_, _ = parentsQuery.WriteString("sharedWithMe=true")
+		if (f.opt.SharedWithMe || f.opt.StarredOnly) && dirID == f.rootFolderID {
+			if f.opt.SharedWithMe {
+				_, _ = parentsQuery.WriteString("sharedWithMe=true")
+			}
+			if f.opt.StarredOnly {
+				if f.opt.SharedWithMe {
+					_, _ = parentsQuery.WriteString(" and ")
+				}
+				_, _ = parentsQuery.WriteString("starred=true")
+			}
 		} else {
 			_, _ = fmt.Fprintf(parentsQuery, "'%s' in parents", dirID)
 		}
@@ -1628,7 +1643,7 @@ func (s listRSlices) Less(i, j int) bool {
 // In each cycle it will read up to grouping entries from the in channel without blocking.
 // If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
 // nil is send to the out channel and the function returns.
-func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listREntry, out chan<- error, cb func(fs.DirEntry) error) {
+func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listREntry, out chan<- error, cb func(fs.DirEntry) error, sendJob func(listREntry)) {
 	var dirs []string
 	var paths []string
 	var grouping int32
@@ -1709,24 +1724,17 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 			if atomic.SwapInt32(&f.grouping, 1) != 1 {
 				fs.Debugf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
 			}
-			var recycled = make([]listREntry, len(dirs))
 			f.listRmu.Lock()
 			for i := range dirs {
-				recycled[i] = listREntry{id: dirs[i], path: paths[i]}
+				// Requeue the jobs
+				job := listREntry{id: dirs[i], path: paths[i]}
+				sendJob(job)
 				// Make a note of these dirs - if they all turn
 				// out to be empty then we can re-enable grouping
 				f.listRempties[dirs[i]] = struct{}{}
 			}
 			f.listRmu.Unlock()
-			// recycle these in the background so we don't deadlock
-			// the listR runners if they all get here
-			wg.Add(len(recycled))
-			go func() {
-				for _, entry := range recycled {
-					in <- entry
-				}
-				fs.Debugf(f, "Recycled %d entries", len(recycled))
-			}()
+			fs.Debugf(f, "Recycled %d entries", len(dirs))
 		}
 		// If using a grouping of 1 and dir was empty then check to see if it
 		// is part of the group that caused grouping to be disabled.
@@ -1795,21 +1803,33 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	overflow := []listREntry{}
 	listed := 0
 
-	cb := func(entry fs.DirEntry) error {
+	// Send a job to the input channel if not closed. If the job
+	// won't fit then queue it in the overflow slice.
+	//
+	// This will not block if the channel is full.
+	sendJob := func(job listREntry) {
 		mu.Lock()
 		defer mu.Unlock()
-		if d, isDir := entry.(*fs.Dir); isDir && in != nil {
+		if in == nil {
+			return
+		}
+		wg.Add(1)
+		select {
+		case in <- job:
+		default:
+			overflow = append(overflow, job)
+			wg.Add(-1)
+		}
+	}
+
+	// Send the entry to the caller, queueing any directories as new jobs
+	cb := func(entry fs.DirEntry) error {
+		if d, isDir := entry.(*fs.Dir); isDir {
 			job := listREntry{actualID(d.ID()), d.Remote()}
-			select {
-			case in <- job:
-				// Adding the wg after we've entered the item is
-				// safe here because we know when the callback
-				// is called we are holding a waitgroup.
-				wg.Add(1)
-			default:
-				overflow = append(overflow, job)
-			}
+			sendJob(job)
 		}
+		mu.Lock()
+		defer mu.Unlock()
 		listed++
 		return list.Add(entry)
 	}
@@ -1818,7 +1838,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	in <- listREntry{directoryID, dir}
 
 	for i := 0; i < fs.Config.Checkers; i++ {
-		go f.listRRunner(ctx, &wg, in, out, cb)
+		go f.listRRunner(ctx, &wg, in, out, cb, sendJob)
 	}
 	go func() {
 		// wait until the all directories are processed
@@ -2258,13 +2278,13 @@ func (f *Fs) Precision() time.Duration {
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	var srcObj *baseObject
 	ext := ""
-	readDescription := false
+	isDoc := false
 	switch src := src.(type) {
 	case *Object:
 		srcObj = &src.baseObject
 	case *documentObject:
 		srcObj, ext = &src.baseObject, src.ext()
-		readDescription = true
+		isDoc = true
 	case *linkObject:
 		srcObj, ext = &src.baseObject, src.ext()
 	default:
@@ -2272,6 +2292,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorCantCopy
 	}
 
+	// Look to see if there is an existing object before we remove
+	// the extension from the remote
+	existingObject, _ := f.NewObject(ctx, remote)
+
+	// Adjust the remote name to be without the extension if we
+	// are about to create a doc.
 	if ext != "" {
 		if !strings.HasSuffix(remote, ext) {
 			fs.Debugf(src, "Can't copy - not same document type")
@@ -2280,15 +2306,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		remote = remote[:len(remote)-len(ext)]
 	}
 
-	// Look to see if there is an existing object
-	existingObject, _ := f.NewObject(ctx, remote)
-
 	createInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
 	if err != nil {
 		return nil, err
 	}
 
-	if readDescription {
+	if isDoc {
 		// preserve the description on copy for docs
 		info, err := f.getFile(actualID(srcObj.id), "description")
 		if err != nil {
@@ -2320,6 +2343,22 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
+	// Google docs aren't preserving their mod time after copy, so set them explicitly
+	// See: https://github.com/rclone/rclone/issues/4517
+	//
+	// FIXME remove this when google fixes the problem!
+	if isDoc {
+		// A short sleep is needed here in order to make the
+		// change effective, without it is is ignored. This is
+		// probably some eventual consistency nastiness.
+		sleepTime := 2 * time.Second
+		fs.Debugf(f, "Sleeping for %v before setting the modtime to work around drive bug - see #4517", sleepTime)
+		time.Sleep(sleepTime)
+		err = newObject.SetModTime(ctx, src.ModTime(ctx))
+		if err != nil {
+			return nil, err
+		}
+	}
 	if existingObject != nil {
 		err = existingObject.Remove(ctx)
 		if err != nil {
diff --git a/backend/fichier/api.go b/backend/fichier/api.go
index c67d05259..483f8c62c 100644
--- a/backend/fichier/api.go
+++ b/backend/fichier/api.go
@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"regexp"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -28,6 +29,20 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
+	// Detect this error which the integration tests provoke
+	// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
+	//
+	// https://1fichier.com/api.html
+	//
+	// file/ls.cgi is limited :
+	//
+	// Warning (can be changed in case of abuses) :
+	// List all files of the account is limited to 1 request per hour.
+	// List folders is limited to 5 000 results and 1 request per folder per 30s.
+	if err != nil && strings.Contains(err.Error(), "Flood detected") {
+		fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
+		time.Sleep(30 * time.Second)
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go
index 10eb955e4..2f80b336c 100644
--- a/backend/jottacloud/jottacloud.go
+++ b/backend/jottacloud/jottacloud.go
@@ -353,7 +353,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
 				authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
 				opts.ExtraHeaders = make(map[string]string)
 				opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
-				resp, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
+				_, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
 			}
 		}
 	}
diff --git a/backend/local/aaaa b/backend/local/aaaa
deleted file mode 100644
index e69de29bb..000000000
diff --git a/backend/local/about_unix.go b/backend/local/about_unix.go
index f856a7964..3c31d17a1 100644
--- a/backend/local/about_unix.go
+++ b/backend/local/about_unix.go
@@ -4,6 +4,7 @@ package local
 
 import (
 	"context"
+	"os"
 	"syscall"
 
 	"github.com/pkg/errors"
@@ -15,6 +16,9 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var s syscall.Statfs_t
 	err := syscall.Statfs(f.root, &s)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, fs.ErrorDirNotFound
+		}
 		return nil, errors.Wrap(err, "failed to read disk usage")
 	}
 	bs := int64(s.Bsize) // nolint: unconvert
diff --git a/backend/local/local.go b/backend/local/local.go
index 90aad3c96..887c66725 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -625,6 +625,10 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	dir = f.localPath(dir)
 	fi, err := f.lstat(dir)
 	if err != nil {
+		// already purged
+		if os.IsNotExist(err) {
+			return fs.ErrorDirNotFound
+		}
 		return err
 	}
 	if !fi.Mode().IsDir() {
diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go
index 42a4e5b3e..9254b5a69 100644
--- a/backend/pcloud/pcloud.go
+++ b/backend/pcloud/pcloud.go
@@ -122,9 +122,19 @@ func init() {
 			Name: "hostname",
 			Help: `Hostname to connect to.
 
-This is normally set when rclone initially does the oauth connection.`,
+This is normally set when rclone initially does the oauth connection,
+however you will need to set it by hand if you are using remote config
+with rclone authorize.
+`,
 			Default:  defaultHostname,
 			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: defaultHostname,
+				Help:  "Original/US region",
+			}, {
+				Value: "eapi.pcloud.com",
+				Help:  "EU region",
+			}},
 		}}...),
 	})
 }
@@ -814,14 +824,19 @@ func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (str
 }
 
 func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (string, error) {
+	obj, err := f.NewObject(ctx, path)
+	if err != nil {
+		return "", err
+	}
+	o := obj.(*Object)
 	opts := rest.Opts{
 		Method:     "POST",
 		Path:       "/getfilepublink",
 		Parameters: url.Values{},
 	}
 	var result api.PubLinkResult
-	opts.Parameters.Set("path", path)
-	err := f.pacer.Call(func() (bool, error) {
+	opts.Parameters.Set("fileid", fileIDtoNumber(o.id))
+	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
 		return shouldRetry(resp, err)
@@ -834,11 +849,6 @@ func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (str
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
 func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
-	err := f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return "", err
-	}
-
 	dirID, err := f.dirCache.FindDir(ctx, remote, false)
 	if err == fs.ErrorDirNotFound {
 		return f.linkFile(ctx, remote, expire)
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 10b0bc525..64f09b267 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -202,107 +202,191 @@ func init() {
 			Help:     "Endpoint for IBM COS S3 API.\nSpecify if using an IBM COS On Premise.",
 			Provider: "IBMCOS",
 			Examples: []fs.OptionExample{{
-				Value: "s3-api.us-geo.objectstorage.softlayer.net",
+				Value: "s3.us.cloud-object-storage.appdomain.cloud",
 				Help:  "US Cross Region Endpoint",
 			}, {
-				Value: "s3-api.dal.us-geo.objectstorage.softlayer.net",
+				Value: "s3.dal.us.cloud-object-storage.appdomain.cloud",
 				Help:  "US Cross Region Dallas Endpoint",
 			}, {
-				Value: "s3-api.wdc-us-geo.objectstorage.softlayer.net",
+				Value: "s3.wdc.us.cloud-object-storage.appdomain.cloud",
 				Help:  "US Cross Region Washington DC Endpoint",
 			}, {
-				Value: "s3-api.sjc-us-geo.objectstorage.softlayer.net",
+				Value: "s3.sjc.us.cloud-object-storage.appdomain.cloud",
 				Help:  "US Cross Region San Jose Endpoint",
 			}, {
-				Value: "s3-api.us-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.us.cloud-object-storage.appdomain.cloud",
 				Help:  "US Cross Region Private Endpoint",
 			}, {
-				Value: "s3-api.dal-us-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.dal.us.cloud-object-storage.appdomain.cloud",
 				Help:  "US Cross Region Dallas Private Endpoint",
 			}, {
-				Value: "s3-api.wdc-us-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.wdc.us.cloud-object-storage.appdomain.cloud",
 				Help:  "US Cross Region Washington DC Private Endpoint",
 			}, {
-				Value: "s3-api.sjc-us-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.sjc.us.cloud-object-storage.appdomain.cloud",
 				Help:  "US Cross Region San Jose Private Endpoint",
 			}, {
-				Value: "s3.us-east.objectstorage.softlayer.net",
+				Value: "s3.us-east.cloud-object-storage.appdomain.cloud",
 				Help:  "US Region East Endpoint",
 			}, {
-				Value: "s3.us-east.objectstorage.service.networklayer.com",
+				Value: "s3.private.us-east.cloud-object-storage.appdomain.cloud",
 				Help:  "US Region East Private Endpoint",
 			}, {
-				Value: "s3.us-south.objectstorage.softlayer.net",
+				Value: "s3.us-south.cloud-object-storage.appdomain.cloud",
 				Help:  "US Region South Endpoint",
 			}, {
-				Value: "s3.us-south.objectstorage.service.networklayer.com",
+				Value: "s3.private.us-south.cloud-object-storage.appdomain.cloud",
 				Help:  "US Region South Private Endpoint",
 			}, {
-				Value: "s3.eu-geo.objectstorage.softlayer.net",
+				Value: "s3.eu.cloud-object-storage.appdomain.cloud",
 				Help:  "EU Cross Region Endpoint",
 			}, {
-				Value: "s3.fra-eu-geo.objectstorage.softlayer.net",
+				Value: "s3.fra.eu.cloud-object-storage.appdomain.cloud",
 				Help:  "EU Cross Region Frankfurt Endpoint",
 			}, {
-				Value: "s3.mil-eu-geo.objectstorage.softlayer.net",
+				Value: "s3.mil.eu.cloud-object-storage.appdomain.cloud",
 				Help:  "EU Cross Region Milan Endpoint",
 			}, {
-				Value: "s3.ams-eu-geo.objectstorage.softlayer.net",
+				Value: "s3.ams.eu.cloud-object-storage.appdomain.cloud",
 				Help:  "EU Cross Region Amsterdam Endpoint",
 			}, {
-				Value: "s3.eu-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.eu.cloud-object-storage.appdomain.cloud",
 				Help:  "EU Cross Region Private Endpoint",
 			}, {
-				Value: "s3.fra-eu-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.fra.eu.cloud-object-storage.appdomain.cloud",
 				Help:  "EU Cross Region Frankfurt Private Endpoint",
 			}, {
-				Value: "s3.mil-eu-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.mil.eu.cloud-object-storage.appdomain.cloud",
 				Help:  "EU Cross Region Milan Private Endpoint",
 			}, {
-				Value: "s3.ams-eu-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.ams.eu.cloud-object-storage.appdomain.cloud",
 				Help:  "EU Cross Region Amsterdam Private Endpoint",
 			}, {
-				Value: "s3.eu-gb.objectstorage.softlayer.net",
+				Value: "s3.eu-gb.cloud-object-storage.appdomain.cloud",
 				Help:  "Great Britain Endpoint",
 			}, {
-				Value: "s3.eu-gb.objectstorage.service.networklayer.com",
+				Value: "s3.private.eu-gb.cloud-object-storage.appdomain.cloud",
 				Help:  "Great Britain Private Endpoint",
 			}, {
-				Value: "s3.ap-geo.objectstorage.softlayer.net",
+				Value: "s3.eu-de.cloud-object-storage.appdomain.cloud",
+				Help:  "EU Region DE Endpoint",
+			}, {
+				Value: "s3.private.eu-de.cloud-object-storage.appdomain.cloud",
+				Help:  "EU Region DE Private Endpoint",
+			}, {
+				Value: "s3.ap.cloud-object-storage.appdomain.cloud",
 				Help:  "APAC Cross Regional Endpoint",
 			}, {
-				Value: "s3.tok-ap-geo.objectstorage.softlayer.net",
+				Value: "s3.tok.ap.cloud-object-storage.appdomain.cloud",
 				Help:  "APAC Cross Regional Tokyo Endpoint",
 			}, {
-				Value: "s3.hkg-ap-geo.objectstorage.softlayer.net",
+				Value: "s3.hkg.ap.cloud-object-storage.appdomain.cloud",
 				Help:  "APAC Cross Regional HongKong Endpoint",
 			}, {
-				Value: "s3.seo-ap-geo.objectstorage.softlayer.net",
+				Value: "s3.seo.ap.cloud-object-storage.appdomain.cloud",
 				Help:  "APAC Cross Regional Seoul Endpoint",
 			}, {
-				Value: "s3.ap-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.ap.cloud-object-storage.appdomain.cloud",
 				Help:  "APAC Cross Regional Private Endpoint",
 			}, {
-				Value: "s3.tok-ap-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.tok.ap.cloud-object-storage.appdomain.cloud",
 				Help:  "APAC Cross Regional Tokyo Private Endpoint",
 			}, {
-				Value: "s3.hkg-ap-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.hkg.ap.cloud-object-storage.appdomain.cloud",
 				Help:  "APAC Cross Regional HongKong Private Endpoint",
 			}, {
-				Value: "s3.seo-ap-geo.objectstorage.service.networklayer.com",
+				Value: "s3.private.seo.ap.cloud-object-storage.appdomain.cloud",
 				Help:  "APAC Cross Regional Seoul Private Endpoint",
 			}, {
-				Value: "s3.mel01.objectstorage.softlayer.net",
+				Value: "s3.jp-tok.cloud-object-storage.appdomain.cloud",
+				Help:  "APAC Region Japan Endpoint",
+			}, {
+				Value: "s3.private.jp-tok.cloud-object-storage.appdomain.cloud",
+				Help:  "APAC Region Japan Private Endpoint",
+			}, {
+				Value: "s3.au-syd.cloud-object-storage.appdomain.cloud",
+				Help:  "APAC Region Australia Endpoint",
+			}, {
+				Value: "s3.private.au-syd.cloud-object-storage.appdomain.cloud",
+				Help:  "APAC Region Australia Private Endpoint",
+			}, {
+				Value: "s3.ams03.cloud-object-storage.appdomain.cloud",
+				Help:  "Amsterdam Single Site Endpoint",
+			}, {
+				Value: "s3.private.ams03.cloud-object-storage.appdomain.cloud",
+				Help:  "Amsterdam Single Site Private Endpoint",
+			}, {
+				Value: "s3.che01.cloud-object-storage.appdomain.cloud",
+				Help:  "Chennai Single Site Endpoint",
+			}, {
+				Value: "s3.private.che01.cloud-object-storage.appdomain.cloud",
+				Help:  "Chennai Single Site Private Endpoint",
+			}, {
+				Value: "s3.mel01.cloud-object-storage.appdomain.cloud",
 				Help:  "Melbourne Single Site Endpoint",
 			}, {
-				Value: "s3.mel01.objectstorage.service.networklayer.com",
+				Value: "s3.private.mel01.cloud-object-storage.appdomain.cloud",
 				Help:  "Melbourne Single Site Private Endpoint",
 			}, {
-				Value: "s3.tor01.objectstorage.softlayer.net",
+				Value: "s3.osl01.cloud-object-storage.appdomain.cloud",
+				Help:  "Oslo Single Site Endpoint",
+			}, {
+				Value: "s3.private.osl01.cloud-object-storage.appdomain.cloud",
+				Help:  "Oslo Single Site Private Endpoint",
+			}, {
+				Value: "s3.tor01.cloud-object-storage.appdomain.cloud",
 				Help:  "Toronto Single Site Endpoint",
 			}, {
-				Value: "s3.tor01.objectstorage.service.networklayer.com",
+				Value: "s3.private.tor01.cloud-object-storage.appdomain.cloud",
 				Help:  "Toronto Single Site Private Endpoint",
+			}, {
+				Value: "s3.seo01.cloud-object-storage.appdomain.cloud",
+				Help:  "Seoul Single Site Endpoint",
+			}, {
+				Value: "s3.private.seo01.cloud-object-storage.appdomain.cloud",
+				Help:  "Seoul Single Site Private Endpoint",
+			}, {
+				Value: "s3.mon01.cloud-object-storage.appdomain.cloud",
+				Help:  "Montreal Single Site Endpoint",
+			}, {
+				Value: "s3.private.mon01.cloud-object-storage.appdomain.cloud",
+				Help:  "Montreal Single Site Private Endpoint",
+			}, {
+				Value: "s3.mex01.cloud-object-storage.appdomain.cloud",
+				Help:  "Mexico Single Site Endpoint",
+			}, {
+				Value: "s3.private.mex01.cloud-object-storage.appdomain.cloud",
+				Help:  "Mexico Single Site Private Endpoint",
+			}, {
+				Value: "s3.sjc04.cloud-object-storage.appdomain.cloud",
+				Help:  "San Jose Single Site Endpoint",
+			}, {
+				Value: "s3.private.sjc04.cloud-object-storage.appdomain.cloud",
+				Help:  "San Jose Single Site Private Endpoint",
+			}, {
+				Value: "s3.mil01.cloud-object-storage.appdomain.cloud",
+				Help:  "Milan Single Site Endpoint",
+			}, {
+				Value: "s3.private.mil01.cloud-object-storage.appdomain.cloud",
+				Help:  "Milan Single Site Private Endpoint",
+			}, {
+				Value: "s3.hkg02.cloud-object-storage.appdomain.cloud",
+				Help:  "Hong Kong Single Site Endpoint",
+			}, {
+				Value: "s3.private.hkg02.cloud-object-storage.appdomain.cloud",
+				Help:  "Hong Kong Single Site Private Endpoint",
+			}, {
+				Value: "s3.par01.cloud-object-storage.appdomain.cloud",
+				Help:  "Paris Single Site Endpoint",
+			}, {
+				Value: "s3.private.par01.cloud-object-storage.appdomain.cloud",
+				Help:  "Paris Single Site Private Endpoint",
+			}, {
+				Value: "s3.sng01.cloud-object-storage.appdomain.cloud",
+				Help:  "Singapore Single Site Endpoint",
+			}, {
+				Value: "s3.private.sng01.cloud-object-storage.appdomain.cloud",
+				Help:  "Singapore Single Site Private Endpoint",
 			}},
 		}, {
 			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
@@ -980,10 +1064,12 @@ This option controls how often unused buffers will be removed from the pool.`,
 
 // Constants
 const (
-	metaMtime           = "Mtime"                // the meta key to store mtime in - eg X-Amz-Meta-Mtime
-	metaMD5Hash         = "Md5chksum"            // the meta key to store md5hash in
-	maxSizeForCopy      = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY
-	maxUploadParts      = 10000                  // maximum allowed number of parts in a multi-part upload
+	metaMtime   = "Mtime"     // the meta key to store mtime in - eg X-Amz-Meta-Mtime
+	metaMD5Hash = "Md5chksum" // the meta key to store md5hash in
+	// The maximum size of object we can COPY - this should be 5GiB but is < 5GB for b2 compatibility
+	// See https://forum.rclone.org/t/copying-files-within-a-b2-bucket/16680/76
+	maxSizeForCopy      = 4768 * 1024 * 1024
+	maxUploadParts      = 10000 // maximum allowed number of parts in a multi-part upload
 	minChunkSize        = fs.SizeSuffix(1024 * 1024 * 5)
 	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
 	maxUploadCutoff     = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
@@ -1883,7 +1969,7 @@ func pathEscape(s string) string {
 //
 // It adds the boiler plate to the req passed in and calls the s3
 // method
-func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, srcSize int64) error {
+func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, src *Object) error {
 	req.Bucket = &dstBucket
 	req.ACL = &f.opt.ACL
 	req.Key = &dstPath
@@ -1899,8 +1985,8 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa
 		req.StorageClass = &f.opt.StorageClass
 	}
 
-	if srcSize >= int64(f.opt.CopyCutoff) {
-		return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, srcSize)
+	if src.bytes >= int64(f.opt.CopyCutoff) {
+		return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, src)
 	}
 	return f.pacer.Call(func() (bool, error) {
 		_, err := f.c.CopyObjectWithContext(ctx, req)
@@ -1921,14 +2007,33 @@ func calculateRange(partSize, partIndex, numParts, totalSize int64) string {
 	return fmt.Sprintf("bytes=%v-%v", start, ends)
 }
 
-func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, srcSize int64) (err error) {
+func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, src *Object) (err error) {
+	info, err := src.headObject(ctx)
+	if err != nil {
+		return err
+	}
+
+	req := &s3.CreateMultipartUploadInput{}
+
+	// Fill in the request from the head info
+	structs.SetFrom(req, info)
+
+	// If copy metadata was set then set the Metadata to that read
+	// from the head request
+	if aws.StringValue(copyReq.MetadataDirective) == s3.MetadataDirectiveCopy {
+		copyReq.Metadata = info.Metadata
+	}
+
+	// Overwrite any from the copyReq
+	structs.SetFrom(req, copyReq)
+
+	req.Bucket = &dstBucket
+	req.Key = &dstPath
+
 	var cout *s3.CreateMultipartUploadOutput
 	if err := f.pacer.Call(func() (bool, error) {
 		var err error
-		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
-			Bucket: &dstBucket,
-			Key:    &dstPath,
-		})
+		cout, err = f.c.CreateMultipartUploadWithContext(ctx, req)
 		return f.shouldRetry(err)
 	}); err != nil {
 		return err
@@ -1937,7 +2042,7 @@ func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBuck
 
 	defer atexit.OnError(&err, func() {
 		// Try to abort the upload, but ignore the error.
-		fs.Debugf(nil, "Cancelling multipart copy")
+		fs.Debugf(src, "Cancelling multipart copy")
 		_ = f.pacer.Call(func() (bool, error) {
 			_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
 				Bucket:       &dstBucket,
@@ -1949,33 +2054,23 @@ func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBuck
 		})
 	})()
 
+	srcSize := src.bytes
 	partSize := int64(f.opt.CopyCutoff)
 	numParts := (srcSize-1)/partSize + 1
 
+	fs.Debugf(src, "Starting  multipart copy with %d parts", numParts)
+
 	var parts []*s3.CompletedPart
 	for partNum := int64(1); partNum <= numParts; partNum++ {
 		if err := f.pacer.Call(func() (bool, error) {
 			partNum := partNum
-			uploadPartReq := &s3.UploadPartCopyInput{
-				Bucket:          &dstBucket,
-				Key:             &dstPath,
-				PartNumber:      &partNum,
-				UploadId:        uid,
-				CopySourceRange: aws.String(calculateRange(partSize, partNum-1, numParts, srcSize)),
-				// Args copy from req
-				CopySource:                     req.CopySource,
-				CopySourceIfMatch:              req.CopySourceIfMatch,
-				CopySourceIfModifiedSince:      req.CopySourceIfModifiedSince,
-				CopySourceIfNoneMatch:          req.CopySourceIfNoneMatch,
-				CopySourceIfUnmodifiedSince:    req.CopySourceIfUnmodifiedSince,
-				CopySourceSSECustomerAlgorithm: req.CopySourceSSECustomerAlgorithm,
-				CopySourceSSECustomerKey:       req.CopySourceSSECustomerKey,
-				CopySourceSSECustomerKeyMD5:    req.CopySourceSSECustomerKeyMD5,
-				RequestPayer:                   req.RequestPayer,
-				SSECustomerAlgorithm:           req.SSECustomerAlgorithm,
-				SSECustomerKey:                 req.SSECustomerKey,
-				SSECustomerKeyMD5:              req.SSECustomerKeyMD5,
-			}
+			uploadPartReq := &s3.UploadPartCopyInput{}
+			structs.SetFrom(uploadPartReq, copyReq)
+			uploadPartReq.Bucket = &dstBucket
+			uploadPartReq.Key = &dstPath
+			uploadPartReq.PartNumber = &partNum
+			uploadPartReq.UploadId = uid
+			uploadPartReq.CopySourceRange = aws.String(calculateRange(partSize, partNum-1, numParts, srcSize))
 			uout, err := f.c.UploadPartCopyWithContext(ctx, uploadPartReq)
 			if err != nil {
 				return f.shouldRetry(err)
@@ -2028,7 +2123,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	req := s3.CopyObjectInput{
 		MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
 	}
-	err = f.copy(ctx, &req, dstBucket, dstPath, srcBucket, srcPath, srcObj.Size())
+	err = f.copy(ctx, &req, dstBucket, dstPath, srcBucket, srcPath, srcObj)
 	if err != nil {
 		return nil, err
 	}
@@ -2425,19 +2520,12 @@ func (o *Object) Size() int64 {
 	return o.bytes
 }
 
-// readMetaData gets the metadata if it hasn't already been fetched
-//
-// it also sets the info
-func (o *Object) readMetaData(ctx context.Context) (err error) {
-	if o.meta != nil {
-		return nil
-	}
+func (o *Object) headObject(ctx context.Context) (resp *s3.HeadObjectOutput, err error) {
 	bucket, bucketPath := o.split()
 	req := s3.HeadObjectInput{
 		Bucket: &bucket,
 		Key:    &bucketPath,
 	}
-	var resp *s3.HeadObjectOutput
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.HeadObjectWithContext(ctx, &req)
@@ -2446,17 +2534,26 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
 			if awsErr.StatusCode() == http.StatusNotFound {
-				// NotFound indicates bucket was OK
-				// NoSuchBucket would be returned if bucket was bad
-				if awsErr.Code() == "NotFound" {
-					o.fs.cache.MarkOK(bucket)
-				}
-				return fs.ErrorObjectNotFound
+				return nil, fs.ErrorObjectNotFound
 			}
 		}
-		return err
+		return nil, err
 	}
 	o.fs.cache.MarkOK(bucket)
+	return resp, nil
+}
+
+// readMetaData gets the metadata if it hasn't already been fetched
+//
+// it also sets the info
+func (o *Object) readMetaData(ctx context.Context) (err error) {
+	if o.meta != nil {
+		return nil
+	}
+	resp, err := o.headObject(ctx)
+	if err != nil {
+		return err
+	}
 	var size int64
 	// Ignore missing Content-Length assuming it is 0
 	// Some versions of ceph do this due their apache proxies
@@ -2527,7 +2624,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 		Metadata:          o.meta,
 		MetadataDirective: aws.String(s3.MetadataDirectiveReplace), // replace metadata with that passed in
 	}
-	return o.fs.copy(ctx, &req, bucket, bucketPath, bucket, bucketPath, o.bytes)
+	return o.fs.copy(ctx, &req, bucket, bucketPath, bucket, bucketPath, o)
 }
 
 // Storable raturns a boolean indicating if this object is storable
@@ -2967,7 +3064,7 @@ func (o *Object) SetTier(tier string) (err error) {
 		MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
 		StorageClass:      aws.String(tier),
 	}
-	err = o.fs.copy(ctx, &req, bucket, bucketPath, bucket, bucketPath, o.bytes)
+	err = o.fs.copy(ctx, &req, bucket, bucketPath, bucket, bucketPath, o)
 	if err != nil {
 		return err
 	}
diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index fceea7a3d..4ab8ce4b0 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -164,6 +164,18 @@ Home directory can be found in a shared folder called "home"
 			Default:  false,
 			Help:     "Set to skip any symlinks and any other non regular files.",
 			Advanced: true,
+		}, {
+			Name:     "subsystem",
+			Default:  "sftp",
+			Help:     "Specifies the SSH2 subsystem on the remote host.",
+			Advanced: true,
+		}, {
+			Name:    "server_command",
+			Default: "",
+			Help: `Specifies the path or command to run a sftp server on the remote host.
+
+The subsystem option is ignored when server_command is defined.`,
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -187,6 +199,8 @@ type Options struct {
 	Md5sumCommand     string `config:"md5sum_command"`
 	Sha1sumCommand    string `config:"sha1sum_command"`
 	SkipLinks         bool   `config:"skip_links"`
+	Subsystem         string `config:"subsystem"`
+	ServerCommand     string `config:"server_command"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -290,7 +304,7 @@ func (f *Fs) sftpConnection() (c *conn, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't connect SSH")
 	}
-	c.sftpClient, err = sftp.NewClient(c.sshClient)
+	c.sftpClient, err = f.newSftpClient(c.sshClient)
 	if err != nil {
 		_ = c.sshClient.Close()
 		return nil, errors.Wrap(err, "couldn't initialise SFTP")
@@ -299,6 +313,35 @@ func (f *Fs) sftpConnection() (c *conn, err error) {
 	return c, nil
 }
 
+// Creates a new SFTP client on conn, using the specified subsystem
+// or sftp server, and zero or more option functions
+func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.Client, error) {
+	s, err := conn.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	pw, err := s.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+	pr, err := s.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	if f.opt.ServerCommand != "" {
+		if err := s.Start(f.opt.ServerCommand); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := s.RequestSubsystem(f.opt.Subsystem); err != nil {
+			return nil, err
+		}
+	}
+
+	return sftp.NewClientPipe(pr, pw, opts...)
+}
+
 // Get an SFTP connection from the pool, or open a new one
 func (f *Fs) getSftpConnection() (c *conn, err error) {
 	f.poolMu.Lock()
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index c44b7feec..08e81d899 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -1335,7 +1335,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// object has been safely uploaded
 		o.lastModified = modTime
 		o.size = size
-		o.md5 = rxHeaders["ETag"]
+		o.md5 = rxHeaders["Etag"]
 		o.contentType = contentType
 		o.headers = headers
 		if inCount != nil {
diff --git a/backend/union/entry.go b/backend/union/entry.go
index 2f5b38b0d..93184399c 100644
--- a/backend/union/entry.go
+++ b/backend/union/entry.go
@@ -1,7 +1,6 @@
 package union
 
 import (
-	"bufio"
 	"context"
 	"io"
 	"sync"
@@ -67,31 +66,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		obj := entries[0].(*upstream.Object)
 		return obj.Update(ctx, in, src, options...)
 	}
-	// Get multiple reader
-	readers := make([]io.Reader, len(entries))
-	writers := make([]io.Writer, len(entries))
-	errs := Errors(make([]error, len(entries)+1))
-	for i := range entries {
-		r, w := io.Pipe()
-		bw := bufio.NewWriter(w)
-		readers[i], writers[i] = r, bw
-		defer func() {
-			err := w.Close()
-			if err != nil {
-				panic(err)
-			}
-		}()
-	}
-	go func() {
-		mw := io.MultiWriter(writers...)
-		es := make([]error, len(writers)+1)
-		_, es[len(es)-1] = io.Copy(mw, in)
-		for i, bw := range writers {
-			es[i] = bw.(*bufio.Writer).Flush()
-		}
-		errs[len(entries)] = Errors(es).Err()
-	}()
 	// Multi-threading
+	readers, errChan := multiReader(len(entries), in)
+	errs := Errors(make([]error, len(entries)+1))
 	multithread(len(entries), func(i int) {
 		if o, ok := entries[i].(*upstream.Object); ok {
 			err := o.Update(ctx, readers[i], src, options...)
@@ -100,6 +77,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			errs[i] = fs.ErrorNotAFile
 		}
 	})
+	errs[len(entries)] = <-errChan
 	return errs.Err()
 }
 
diff --git a/backend/union/union.go b/backend/union/union.go
index a803a7e51..17e988956 100644
--- a/backend/union/union.go
+++ b/backend/union/union.go
@@ -181,6 +181,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	errs := Errors(make([]error, len(upstreams)))
 	multithread(len(upstreams), func(i int) {
 		err := upstreams[i].Features().Purge(ctx, dir)
+		if errors.Cause(err) == fs.ErrorDirNotFound {
+			err = nil
+		}
 		errs[i] = errors.Wrap(err, upstreams[i].Name())
 	})
 	return errs.Err()
@@ -382,6 +385,37 @@ func (f *Fs) DirCacheFlush() {
 	})
 }
 
+// Tee in into n outputs
+//
+// When finished read the error from the channel
+func multiReader(n int, in io.Reader) ([]io.Reader, <-chan error) {
+	readers := make([]io.Reader, n)
+	pipeWriters := make([]*io.PipeWriter, n)
+	writers := make([]io.Writer, n)
+	errChan := make(chan error, 1)
+	for i := range writers {
+		r, w := io.Pipe()
+		bw := bufio.NewWriter(w)
+		readers[i], pipeWriters[i], writers[i] = r, w, bw
+	}
+	go func() {
+		mw := io.MultiWriter(writers...)
+		es := make([]error, 2*n+1)
+		_, copyErr := io.Copy(mw, in)
+		es[2*n] = copyErr
+		// Flush the buffers
+		for i, bw := range writers {
+			es[i] = bw.(*bufio.Writer).Flush()
+		}
+		// Close the underlying pipes
+		for i, pw := range pipeWriters {
+			es[2*i] = pw.CloseWithError(copyErr)
+		}
+		errChan <- Errors(es).Err()
+	}()
+	return readers, errChan
+}
+
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
 	srcPath := src.Remote()
 	upstreams, err := f.create(ctx, srcPath)
@@ -409,31 +443,9 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 		e, err := f.wrapEntries(u.WrapObject(o))
 		return e.(*Object), err
 	}
-	errs := Errors(make([]error, len(upstreams)+1))
-	// Get multiple reader
-	readers := make([]io.Reader, len(upstreams))
-	writers := make([]io.Writer, len(upstreams))
-	for i := range writers {
-		r, w := io.Pipe()
-		bw := bufio.NewWriter(w)
-		readers[i], writers[i] = r, bw
-		defer func() {
-			err := w.Close()
-			if err != nil {
-				panic(err)
-			}
-		}()
-	}
-	go func() {
-		mw := io.MultiWriter(writers...)
-		es := make([]error, len(writers)+1)
-		_, es[len(es)-1] = io.Copy(mw, in)
-		for i, bw := range writers {
-			es[i] = bw.(*bufio.Writer).Flush()
-		}
-		errs[len(upstreams)] = Errors(es).Err()
-	}()
 	// Multi-threading
+	readers, errChan := multiReader(len(upstreams), in)
+	errs := Errors(make([]error, len(upstreams)+1))
 	objs := make([]upstream.Entry, len(upstreams))
 	multithread(len(upstreams), func(i int) {
 		u := upstreams[i]
@@ -450,6 +462,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 		}
 		objs[i] = u.WrapObject(o)
 	})
+	errs[len(upstreams)] = <-errChan
 	err = errs.Err()
 	if err != nil {
 		return nil, err
@@ -504,6 +517,9 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	}
 	for _, u := range f.upstreams {
 		usg, err := u.About(ctx)
+		if errors.Cause(err) == fs.ErrorDirNotFound {
+			continue
+		}
 		if err != nil {
 			return nil, err
 		}
diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go
index 2a58048e8..49a993b28 100644
--- a/backend/union/upstream/upstream.go
+++ b/backend/union/upstream/upstream.go
@@ -97,6 +97,7 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) {
 		return nil, err
 	}
 	f.Fs = myFs
+	cache.PinUntilFinalized(f.Fs, f)
 	return f, err
 }
 
@@ -335,6 +336,9 @@ func (f *Fs) updateUsageCore(lock bool) error {
 	usage, err := f.RootFs.Features().About(ctx)
 	if err != nil {
 		f.cacheUpdate = false
+		if errors.Cause(err) == fs.ErrorDirNotFound {
+			err = nil
+		}
 		return err
 	}
 	if lock {
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 1d606a4d4..8b06b096a 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -89,8 +89,10 @@ func NewFsFile(remote string) (fs.Fs, string) {
 	f, err := cache.Get(remote)
 	switch err {
 	case fs.ErrorIsFile:
+		cache.Pin(f) // pin indefinitely since it was on the CLI
 		return f, path.Base(fsPath)
 	case nil:
+		cache.Pin(f) // pin indefinitely since it was on the CLI
 		return f, ""
 	default:
 		err = fs.CountError(err)
@@ -139,6 +141,7 @@ func newFsDir(remote string) fs.Fs {
 		err = fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
 	}
+	cache.Pin(f) // pin indefinitely since it was on the CLI
 	return f
 }
 
@@ -197,6 +200,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 		_ = fs.CountError(err)
 		log.Fatalf("Failed to create file system for destination %q: %v", dstRemote, err)
 	}
+	cache.Pin(fdst) // pin indefinitely since it was on the CLI
 	return
 }
 
diff --git a/cmd/version/version.go b/cmd/version/version.go
index a98f459de..04f157e7f 100644
--- a/cmd/version/version.go
+++ b/cmd/version/version.go
@@ -7,11 +7,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/flags"
-	"github.com/rclone/rclone/fs/version"
 	"github.com/spf13/cobra"
 )
 
@@ -66,8 +66,16 @@ Or
 	},
 }
 
+// strip a leading v off the string
+func stripV(s string) string {
+	if len(s) > 0 && s[0] == 'v' {
+		return s[1:]
+	}
+	return s
+}
+
 // getVersion gets the version by checking the download repository passed in
-func getVersion(url string) (v version.Version, vs string, date time.Time, err error) {
+func getVersion(url string) (v *semver.Version, vs string, date time.Time, err error) {
 	resp, err := http.Get(url)
 	if err != nil {
 		return v, vs, date, err
@@ -89,16 +97,16 @@ func getVersion(url string) (v version.Version, vs string, date time.Time, err e
 	if err != nil {
 		return v, vs, date, err
 	}
-	v, err = version.New(vs)
+	v, err = semver.NewVersion(stripV(vs))
 	return v, vs, date, err
 }
 
 // check the current version against available versions
 func checkVersion() {
 	// Get Current version
-	vCurrent, err := version.New(fs.Version)
+	vCurrent, err := semver.NewVersion(stripV(fs.Version))
 	if err != nil {
-		fs.Errorf(nil, "Failed to get parse version: %v", err)
+		fs.Errorf(nil, "Failed to parse version: %v", err)
 	}
 	const timeFormat = "2006-01-02"
 
@@ -108,12 +116,12 @@ func checkVersion() {
 			fs.Errorf(nil, "Failed to get rclone %s version: %v", what, err)
 			return
 		}
-		fmt.Printf("%-8s%-13v %20s\n",
+		fmt.Printf("%-8s%-40v %20s\n",
 			what+":",
 			v,
 			"(released "+t.Format(timeFormat)+")",
 		)
-		if v.Cmp(vCurrent) > 0 {
+		if v.Compare(*vCurrent) > 0 {
 			fmt.Printf("  upgrade: %s\n", url+vs)
 		}
 	}
@@ -126,7 +134,7 @@ func checkVersion() {
 		"beta",
 		"https://beta.rclone.org/",
 	)
-	if vCurrent.IsGit() {
+	if strings.HasSuffix(fs.Version, "-DEV") {
 		fmt.Println("Your version is compiled from git so comparisons may be wrong.")
 	}
 }
diff --git a/docs/content/authors.md b/docs/content/authors.md
index b1dde58ef..a6558d05a 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -400,3 +400,10 @@ put them back in again.` >}}
   * tyhuber1 <68970760+tyhuber1@users.noreply.github.com>
   * David Ibarra <david.ibarra@realty.com>
   * Tim Gallant <tim@lilt.com>
+  * Kaloyan Raev <kaloyan@storj.io>
+  * Jay McEntire <jay.mcentire@gmail.com>
+  * Leo Luan <leoluan@us.ibm.com>
+  * aus <549081+aus@users.noreply.github.com>
+  * Aaron Gokaslan <agokaslan@fb.com>
+  * Egor Margineanu <egmar@users.noreply.github.com>
+  * Lucas Kanashiro <lucas.kanashiro@canonical.com>
diff --git a/docs/content/downloads.md b/docs/content/downloads.md
index 5537cc0b1..43b42f38b 100644
--- a/docs/content/downloads.md
+++ b/docs/content/downloads.md
@@ -38,16 +38,25 @@ Beta releases
 [Beta releases](https://beta.rclone.org) are generated from each commit
 to master.  Note these are named like
 
-    {Version Tag}-{Commit Number}-g{Git Commit Hash}
+    {Version Tag}.beta.{Commit Number}.{Git Commit Hash}
 
-You can match the `Git Commit Hash` up with the [git
-log](https://github.com/rclone/rclone/commits/master).  The most recent
-release will have the largest `Version Tag` and `Commit Number` and
-will normally be at the end of the list.
+eg
+
+    v1.53.0-beta.4677.b657a2204
+
+The `Version Tag` is the version that the beta release will become
+when it is released. You can match the `Git Commit Hash` up with the
+[git log](https://github.com/rclone/rclone/commits/master).  The most
+recent release will have the largest `Version Tag` and `Commit Number`
+and will normally be at the end of the list.
 
 Some beta releases may have a branch name also:
 
-    {Version Tag}-{Commit Number}-g{Git Commit Hash}-{Branch Name}
+    {Version Tag}-beta.{Commit Number}.{Git Commit Hash}.{Branch Name}
+
+eg
+
+    v1.53.0-beta.4677.b657a2204.semver
 
 The presence of `Branch Name` indicates that this is a feature under
 development which will at some point be merged into the normal betas
diff --git a/docs/content/install.md b/docs/content/install.md
index 2faf957d2..75027571a 100644
--- a/docs/content/install.md
+++ b/docs/content/install.md
@@ -136,6 +136,13 @@ from the rclone image.
   reside on the host with a non-root UID:GID, you need to pass these on the container
   start command line.
 
+- If you want to access the RC interface (either via the API or the Web UI), it is
+  required to set the `--rc-addr` to `:5572` in order to connect to it from outside
+  the container. An explanation about why this is necessary is present [here](https://web.archive.org/web/20200808071950/https://pythonspeed.com/articles/docker-connection-refused/).
+    * NOTE: Users running this container with the docker network set to `host` should
+     probably set it to listen to localhost only, with `127.0.0.1:5572` as the value for
+      `--rc-addr`
+
 - It is possible to use `rclone mount` inside a userspace Docker container, and expose
   the resulting fuse mount to the host. The exact `docker run` options to do that might
   vary slightly between hosts. See, e.g. the discussion in this
diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md
index 36ecf5909..330f7476b 100644
--- a/docs/content/onedrive.md
+++ b/docs/content/onedrive.md
@@ -294,7 +294,12 @@ See: the [encoding section in the overview](/overview/#encoding) for more info.
 
 {{< rem autogenerated options stop >}}
 
-### Limitations ###
+### Limitations
+
+If you don't use rclone for 90 days the refresh token will
+expire. This will result in authorization problems. This is easy to
+fix by running the `rclone config reconnect remote:` command to get a
+new token and refresh token.
 
 #### Naming ####
 
@@ -402,13 +407,24 @@ It is a
 [known](https://github.com/OneDrive/onedrive-api-docs/issues/935#issuecomment-441741631)
 issue that Sharepoint (not OneDrive or OneDrive for Business) silently modifies
 uploaded files, mainly Office files (.docx, .xlsx, etc.), causing file size and
-hash checks to fail. To use rclone with such affected files on Sharepoint, you
+hash checks to fail. There are also other situations that will cause OneDrive to 
+report inconsistent file sizes. To use rclone with such
+affected files on Sharepoint, you
 may disable these checks with the following command line arguments:
 
 ```
 --ignore-checksum --ignore-size
 ```
 
+Alternatively, if you have write access to the OneDrive files, it may be possible
+to fix this problem for certain files, by attempting the steps below.
+Open the web interface for [OneDrive](https://onedrive.live.com) and find the 
+affected files (which will be in the error messages/log for rclone). Simply click on
+each of these files, causing OneDrive to open them on the web. This will cause each 
+file to be converted in place to a format that is functionally equivalent
+but which will no longer trigger the size discrepancy. Once all problematic files
+are converted you will no longer need the ignore options above.
+
 #### Replacing/deleting existing files on Sharepoint gets "item not found" ####
 
 It is a [known](https://github.com/OneDrive/onedrive-api-docs/issues/1068) issue
diff --git a/docs/layouts/partials/version.html b/docs/layouts/partials/version.html
index 00c9d1281..55274d266 100644
--- a/docs/layouts/partials/version.html
+++ b/docs/layouts/partials/version.html
@@ -1,1 +1,1 @@
-v1.52.0
\ No newline at end of file
+v1.53.0
diff --git a/fs/cache/cache.go b/fs/cache/cache.go
index 4a5bdc803..8257ea0dc 100644
--- a/fs/cache/cache.go
+++ b/fs/cache/cache.go
@@ -2,6 +2,7 @@
 package cache
 
 import (
+	"runtime"
 	"sync"
 
 	"github.com/rclone/rclone/fs"
@@ -80,6 +81,18 @@ func Pin(f fs.Fs) {
 	c.Pin(fs.ConfigString(f))
 }
 
+// PinUntilFinalized pins f into the cache until x is garbage collected
+//
+// This calls runtime.SetFinalizer on x so it shouldn't have a
+// finalizer already.
+func PinUntilFinalized(f fs.Fs, x interface{}) {
+	Pin(f)
+	runtime.SetFinalizer(x, func(_ interface{}) {
+		Unpin(f)
+	})
+
+}
+
 // Unpin f from the cache
 func Unpin(f fs.Fs) {
 	c.Pin(fs.ConfigString(f))
diff --git a/fs/fs.go b/fs/fs.go
index c564ec809..53bbdb8a6 100644
--- a/fs/fs.go
+++ b/fs/fs.go
@@ -1330,6 +1330,7 @@ func ConfigFs(path string) (fsInfo *RegInfo, configName, fsPath string, config *
 // On Windows avoid single character remote names as they can be mixed
 // up with drive letters.
 func NewFs(path string) (Fs, error) {
+	Debugf(nil, "Creating backend with remote %q", path)
 	fsInfo, configName, fsPath, config, err := ConfigFs(path)
 	if err != nil {
 		return nil, err
diff --git a/fs/fserrors/enospc_error.go b/fs/fserrors/enospc_error.go
new file mode 100644
index 000000000..a2cb1d496
--- /dev/null
+++ b/fs/fserrors/enospc_error.go
@@ -0,0 +1,23 @@
+// +build !plan9
+
+package fserrors
+
+import (
+	"syscall"
+
+	"github.com/rclone/rclone/lib/errors"
+)
+
+// IsErrNoSpace checks a possibly wrapped error to
+// see if it contains a ENOSPC error
+func IsErrNoSpace(cause error) (isNoSpc bool) {
+	errors.Walk(cause, func(c error) bool {
+		if c == syscall.ENOSPC {
+			isNoSpc = true
+			return true
+		}
+		isNoSpc = false
+		return false
+	})
+	return
+}
diff --git a/fs/fserrors/enospc_error_notsupported.go b/fs/fserrors/enospc_error_notsupported.go
new file mode 100644
index 000000000..c4cd1e940
--- /dev/null
+++ b/fs/fserrors/enospc_error_notsupported.go
@@ -0,0 +1,10 @@
+// +build plan9
+
+package fserrors
+
+// IsErrNoSpace() on plan9 returns false because 
+// plan9 does not support syscall.ENOSPC error.
+func IsErrNoSpace(cause error) (isNoSpc bool) {
+	isNoSpc = false
+	return 
+}
diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index d16c29310..a07b7ceea 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -105,14 +105,21 @@ func Split(remote string) (parent string, leaf string, err error) {
 // JoinRootPath joins any number of path elements into a single path, adding a
 // separating slash if necessary. The result is Cleaned; in particular,
 // all empty strings are ignored.
+//
 // If the first non empty element has a leading "//" this is preserved.
+//
+// If the path contains \ these will be converted to / on Windows.
 func JoinRootPath(elem ...string) string {
-	for i, e := range elem {
+	es := make([]string, len(elem))
+	for i := range es {
+		es[i] = filepath.ToSlash(elem[i])
+	}
+	for i, e := range es {
 		if e != "" {
 			if strings.HasPrefix(e, "//") {
-				return "/" + path.Clean(strings.Join(elem[i:], "/"))
+				return "/" + path.Clean(strings.Join(es[i:], "/"))
 			}
-			return path.Clean(strings.Join(elem[i:], "/"))
+			return path.Clean(strings.Join(es[i:], "/"))
 		}
 	}
 	return ""
diff --git a/fs/rc/internal.go b/fs/rc/internal.go
index a813b006e..24a13b865 100644
--- a/fs/rc/internal.go
+++ b/fs/rc/internal.go
@@ -4,15 +4,18 @@ package rc
 
 import (
 	"context"
+	"net/http"
 	"os"
+	"os/exec"
 	"runtime"
+	"strings"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/pkg/errors"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/version"
 	"github.com/rclone/rclone/lib/atexit"
 )
 
@@ -169,10 +172,10 @@ func init() {
 		Help: `
 This shows the current version of go and the go runtime
 
-- version - rclone version, eg "v1.44"
-- decomposed - version number as [major, minor, patch, subpatch]
-    - note patch and subpatch will be 999 for a git compiled version
+- version - rclone version, eg "v1.53.0"
+- decomposed - version number as [major, minor, patch]
 - isGit - boolean - true if this was compiled from the git version
+- isBeta - boolean - true if this is a beta version
 - os - OS in use as according to Go
 - arch - cpu architecture in use according to Go
 - goVersion - version of Go runtime in use
@@ -183,14 +186,15 @@ This shows the current version of go and the go runtime
 
 // Return version info
 func rcVersion(ctx context.Context, in Params) (out Params, err error) {
-	decomposed, err := version.New(fs.Version)
+	version, err := semver.NewVersion(fs.Version[1:])
 	if err != nil {
 		return nil, err
 	}
 	out = Params{
 		"version":    fs.Version,
-		"decomposed": decomposed,
-		"isGit":      decomposed.IsGit(),
+		"decomposed": version.Slice(),
+		"isGit":      strings.HasSuffix(fs.Version, "-DEV"),
+		"isBeta":     version.PreRelease != "",
 		"os":         runtime.GOOS,
 		"arch":       runtime.GOARCH,
 		"goVersion":  runtime.Version(),
@@ -335,3 +339,135 @@ func rcSetBlockProfileRate(ctx context.Context, in Params) (out Params, err erro
 	runtime.SetBlockProfileRate(int(rate))
 	return nil, nil
 }
+
+func init() {
+	Add(Call{
+		Path:          "core/command",
+		AuthRequired:  true,
+		Fn:            rcRunCommand,
+		NeedsRequest:  true,
+		NeedsResponse: true,
+		Title:         "Run a rclone terminal command over rc.",
+		Help: `This takes the following parameters
+
+- command - a string with the command name
+- arg - a list of arguments for the backend command
+- opt - a map of string to string of options
+
+Returns
+
+- result - result from the backend command
+- error	 - set if rclone exits with an error code
+- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT". "STREAM_ONLY_STDERR")
+
+For example
+
+    rclone rc core/command command=ls -a mydrive:/ -o max-depth=1
+	rclone rc core/command -a ls -a mydrive:/ -o max-depth=1
+
+Returns
+
+` + "```" + `
+{
+	"error": false,
+	"result": "<Raw command line output>"
+}
+
+OR 
+{
+	"error": true,
+	"result": "<Raw command line output>"
+}
+
+
+`,
+	})
+}
+
+// rcRunCommand runs an rclone command with the given args and flags
+func rcRunCommand(ctx context.Context, in Params) (out Params, err error) {
+
+	command, err := in.GetString("command")
+	if err != nil {
+		command = ""
+	}
+
+	var opt = map[string]string{}
+	err = in.GetStructMissingOK("opt", &opt)
+	if err != nil {
+		return nil, err
+	}
+
+	var arg = []string{}
+	err = in.GetStructMissingOK("arg", &arg)
+	if err != nil {
+		return nil, err
+	}
+
+	returnType, err := in.GetString("returnType")
+	if err != nil {
+		returnType = "COMBINED_OUTPUT"
+	}
+
+	var httpResponse *http.ResponseWriter
+	httpResponse, err = in.GetHTTPResponseWriter()
+	if err != nil {
+		return nil, errors.Errorf("response object is required\n" + err.Error())
+	}
+
+	var allArgs = []string{}
+	if command != "" {
+		// Add the command eg: ls to the args
+		allArgs = append(allArgs, command)
+	}
+	// Add all from arg
+	for _, cur := range arg {
+		allArgs = append(allArgs, cur)
+	}
+
+	// Add flags to args for eg --max-depth 1 comes in as { max-depth 1 }.
+	// Convert it to [ max-depth, 1 ] and append to args list
+	for key, value := range opt {
+		if len(key) == 1 {
+			allArgs = append(allArgs, "-"+key)
+		} else {
+			allArgs = append(allArgs, "--"+key)
+		}
+		allArgs = append(allArgs, value)
+	}
+
+	// Get the path for the current executable which was used to run rclone.
+	ex, err := os.Executable()
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := exec.CommandContext(ctx, ex, allArgs...)
+
+	if returnType == "COMBINED_OUTPUT" {
+		// Run the command and get the output for error and stdout combined.
+
+		out, err := cmd.CombinedOutput()
+
+		if err != nil {
+			return Params{
+				"result": string(out),
+				"error":  true,
+			}, nil
+		}
+		return Params{
+			"result": string(out),
+			"error":  false,
+		}, nil
+	} else if returnType == "STREAM_ONLY_STDOUT" {
+		cmd.Stdout = *httpResponse
+	} else if returnType == "STREAM_ONLY_STDERR" {
+		cmd.Stderr = *httpResponse
+	} else if returnType == "STREAM" {
+		cmd.Stdout = *httpResponse
+		cmd.Stderr = *httpResponse
+	}
+
+	err = cmd.Run()
+	return nil, err
+}
diff --git a/fs/rc/params.go b/fs/rc/params.go
index ae7f932e5..bdd020b1e 100644
--- a/fs/rc/params.go
+++ b/fs/rc/params.go
@@ -91,7 +91,7 @@ func (p Params) Get(key string) (interface{}, error) {
 	return value, nil
 }
 
-// GetHTTPRequest gets a http.Request parameter associated with the request with the key "HttpRequest"
+// GetHTTPRequest gets a http.Request parameter associated with the request with the key "_request"
 //
 // If the parameter isn't found then error will be of type
 // ErrParamNotFound and the returned value will be nil.
@@ -108,6 +108,23 @@ func (p Params) GetHTTPRequest() (*http.Request, error) {
 	return request, nil
 }
 
+// GetHTTPResponseWriter gets a http.ResponseWriter parameter associated with the request with the key "_response"
+//
+// If the parameter isn't found then error will be of type
+// ErrParamNotFound and the returned value will be nil.
+func (p Params) GetHTTPResponseWriter() (*http.ResponseWriter, error) {
+	key := "_response"
+	value, err := p.Get(key)
+	if err != nil {
+		return nil, err
+	}
+	request, ok := value.(*http.ResponseWriter)
+	if !ok {
+		return nil, ErrParamInvalid{errors.Errorf("expecting *http.ResponseWriter value for key %q (was %T)", key, value)}
+	}
+	return request, nil
+}
+
 // GetString gets a string parameter from the input
 //
 // If the parameter isn't found then error will be of type
diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go
index 7f324c21b..6204c4810 100644
--- a/fs/rc/rcserver/rcserver.go
+++ b/fs/rc/rcserver/rcserver.go
@@ -17,6 +17,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/rclone/rclone/fs/rc/webgui"
+
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -60,12 +62,14 @@ func Start(opt *rc.Options) (*Server, error) {
 // Server contains everything to run the rc server
 type Server struct {
 	*httplib.Server
-	files http.Handler
-	opt   *rc.Options
+	files          http.Handler
+	pluginsHandler http.Handler
+	opt            *rc.Options
 }
 
 func newServer(opt *rc.Options, mux *http.ServeMux) *Server {
 	fileHandler := http.Handler(nil)
+	pluginsHandler := http.Handler(nil)
 	// Add some more mime types which are often missing
 	_ = mime.AddExtensionType(".wasm", "application/wasm")
 	_ = mime.AddExtensionType(".js", "application/javascript")
@@ -80,7 +84,7 @@ func newServer(opt *rc.Options, mux *http.ServeMux) *Server {
 		fs.Logf(nil, "Serving files from %q", opt.Files)
 		fileHandler = http.FileServer(http.Dir(opt.Files))
 	} else if opt.WebUI {
-		if err := rc.CheckAndDownloadWebGUIRelease(opt.WebGUIUpdate, opt.WebGUIForceUpdate, opt.WebGUIFetchURL, config.CacheDir); err != nil {
+		if err := webgui.CheckAndDownloadWebGUIRelease(opt.WebGUIUpdate, opt.WebGUIForceUpdate, opt.WebGUIFetchURL, config.CacheDir); err != nil {
 			log.Fatalf("Error while fetching the latest release of Web GUI: %v", err)
 		}
 		if opt.NoAuth {
@@ -103,12 +107,15 @@ func newServer(opt *rc.Options, mux *http.ServeMux) *Server {
 
 		fs.Logf(nil, "Serving Web GUI")
 		fileHandler = http.FileServer(http.Dir(extractPath))
+
+		pluginsHandler = http.FileServer(http.Dir(webgui.PluginsPath))
 	}
 
 	s := &Server{
-		Server: httplib.NewServer(mux, &opt.HTTPOptions),
-		opt:    opt,
-		files:  fileHandler,
+		Server:         httplib.NewServer(mux, &opt.HTTPOptions),
+		opt:            opt,
+		files:          fileHandler,
+		pluginsHandler: pluginsHandler,
 	}
 	mux.HandleFunc("/", s.handler)
 
@@ -265,6 +272,10 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
 		in["_request"] = r
 	}
 
+	if call.NeedsResponse {
+		in["_response"] = w
+	}
+
 	// Check to see if it is async or not
 	isAsync, err := in.GetBool("_async")
 	if rc.NotErrParamNotFound(err) {
@@ -361,11 +372,12 @@ var fsMatch = regexp.MustCompile(`^\[(.*?)\](.*)$`)
 
 func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string) {
 	// Look to see if this has an fs in the path
-	match := fsMatch.FindStringSubmatch(path)
+	fsMatchResult := fsMatch.FindStringSubmatch(path)
+
 	switch {
-	case match != nil && s.opt.Serve:
+	case fsMatchResult != nil && s.opt.Serve:
 		// Serve /[fs]/remote files
-		s.serveRemote(w, r, match[2], match[1])
+		s.serveRemote(w, r, fsMatchResult[2], fsMatchResult[1])
 		return
 	case path == "metrics" && s.opt.EnableMetrics:
 		promHandler.ServeHTTP(w, r)
@@ -375,6 +387,19 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
 		s.serveRoot(w, r)
 		return
 	case s.files != nil:
+		pluginsMatchResult := webgui.PluginsMatch.FindStringSubmatch(path)
+
+		if s.opt.WebUI && pluginsMatchResult != nil && len(pluginsMatchResult) > 2 {
+			ok := webgui.ServePluginOK(w, r, pluginsMatchResult)
+			if !ok {
+				r.URL.Path = fmt.Sprintf("/%s/%s/app/build/%s", pluginsMatchResult[1], pluginsMatchResult[2], pluginsMatchResult[3])
+				s.pluginsHandler.ServeHTTP(w, r)
+				return
+			}
+			return
+		} else if s.opt.WebUI && webgui.ServePluginWithReferrerOK(w, r, path) {
+			return
+		}
 		// Serve the files
 		r.URL.Path = "/" + path
 		s.files.ServeHTTP(w, r)
diff --git a/fs/rc/registry.go b/fs/rc/registry.go
index f2f5a1fb8..4cc80541a 100644
--- a/fs/rc/registry.go
+++ b/fs/rc/registry.go
@@ -17,12 +17,13 @@ type Func func(ctx context.Context, in Params) (out Params, err error)
 // Call defines info about a remote control function and is used in
 // the Add function to create new entry points.
 type Call struct {
-	Path         string // path to activate this RC
-	Fn           Func   `json:"-"` // function to call
-	Title        string // help for the function
-	AuthRequired bool   // if set then this call requires authorisation to be set
-	Help         string // multi-line markdown formatted help
-	NeedsRequest bool   // if set then this call will be passed the original request object as _request
+	Path          string // path to activate this RC
+	Fn            Func   `json:"-"` // function to call
+	Title         string // help for the function
+	AuthRequired  bool   // if set then this call requires authorisation to be set
+	Help          string // multi-line markdown formatted help
+	NeedsRequest  bool   // if set then this call will be passed the original request object as _request
+	NeedsResponse bool   // if set then this call will be passed the original response object as _response
 }
 
 // Registry holds the list of all the registered remote control functions
diff --git a/fs/rc/webgui/plugins.go b/fs/rc/webgui/plugins.go
new file mode 100644
index 000000000..fda4896df
--- /dev/null
+++ b/fs/rc/webgui/plugins.go
@@ -0,0 +1,313 @@
+package webgui
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+)
+
+// PackageJSON is the structure of package.json of a plugin
+type PackageJSON struct {
+	Name        string `json:"name"`
+	Version     string `json:"version"`
+	Description string `json:"description"`
+	Author      string `json:"author"`
+	Copyright   string `json:"copyright"`
+	License     string `json:"license"`
+	Private     bool   `json:"private"`
+	Homepage    string `json:"homepage"`
+	TestURL     string `json:"testUrl"`
+	Repository  struct {
+		Type string `json:"type"`
+		URL  string `json:"url"`
+	} `json:"repository"`
+	Bugs struct {
+		URL string `json:"url"`
+	} `json:"bugs"`
+	Rclone RcloneConfig `json:"rclone"`
+}
+
+// RcloneConfig represents the rclone specific config
+type RcloneConfig struct {
+	HandlesType      []string `json:"handlesType"`
+	PluginType       string   `json:"pluginType"`
+	RedirectReferrer bool     `json:"redirectReferrer"`
+	Test             bool     `json:"-"`
+}
+
+func (r *PackageJSON) isTesting() bool {
+	return r.Rclone.Test
+}
+
+var (
+	//loadedTestPlugins *Plugins
+	cachePath string
+
+	loadedPlugins *Plugins
+	pluginsProxy  = &httputil.ReverseProxy{}
+	// PluginsMatch is used for matching author and plugin name in the url path
+	PluginsMatch = regexp.MustCompile(`^plugins\/([^\/]*)\/([^\/\?]+)[\/]?(.*)$`)
+	// PluginsPath is the base path where webgui plugins are stored
+	PluginsPath              string
+	pluginsConfigPath        string
+	availablePluginsJSONPath = "availablePlugins.json"
+)
+
+func init() {
+	cachePath = filepath.Join(config.CacheDir, "webgui")
+	PluginsPath = filepath.Join(cachePath, "plugins")
+	pluginsConfigPath = filepath.Join(PluginsPath, "config")
+
+	loadedPlugins = newPlugins(availablePluginsJSONPath)
+	err := loadedPlugins.readFromFile()
+	if err != nil {
+		fs.Errorf(nil, "error reading available plugins: %v", err)
+	}
+}
+
+// Plugins represents the structure how plugins are saved onto disk
+type Plugins struct {
+	mutex         sync.Mutex
+	LoadedPlugins map[string]PackageJSON `json:"loadedPlugins"`
+	fileName      string
+}
+
+func newPlugins(fileName string) *Plugins {
+	p := Plugins{LoadedPlugins: map[string]PackageJSON{}}
+	p.fileName = fileName
+	p.mutex = sync.Mutex{}
+	return &p
+}
+
+func (p *Plugins) readFromFile() (err error) {
+	//p.mutex.Lock()
+	//defer p.mutex.Unlock()
+	err = CreatePathIfNotExist(pluginsConfigPath)
+	if err != nil {
+		return err
+	}
+	availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName)
+	_, err = os.Stat(availablePluginsJSON)
+	if err == nil {
+		data, err := ioutil.ReadFile(availablePluginsJSON)
+		if err != nil {
+			return err
+		}
+		err = json.Unmarshal(data, &p)
+		if err != nil {
+			fs.Logf(nil, "%s", err)
+		}
+		return nil
+	} else if os.IsNotExist(err) {
+		// path does not exist
+		err = p.writeToFile()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *Plugins) addPlugin(pluginName string, packageJSONPath string) (err error) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	data, err := ioutil.ReadFile(packageJSONPath)
+	if err != nil {
+		return err
+	}
+	var pkgJSON = PackageJSON{}
+	err = json.Unmarshal(data, &pkgJSON)
+	if err != nil {
+		return err
+	}
+	p.LoadedPlugins[pluginName] = pkgJSON
+
+	err = p.writeToFile()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *Plugins) addTestPlugin(pluginName string, testURL string, handlesType []string) (err error) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	err = p.readFromFile()
+	if err != nil {
+		return err
+	}
+
+	var pkgJSON = PackageJSON{
+		Name:    pluginName,
+		TestURL: testURL,
+		Rclone: RcloneConfig{
+			HandlesType: handlesType,
+			Test:        true,
+		},
+	}
+
+	p.LoadedPlugins[pluginName] = pkgJSON
+
+	err = p.writeToFile()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *Plugins) writeToFile() (err error) {
+	//p.mutex.Lock()
+	//defer p.mutex.Unlock()
+	availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName)
+
+	file, err := json.MarshalIndent(p, "", " ")
+	if err != nil {
+		fs.Logf(nil, "%s", err)
+	}
+	err = ioutil.WriteFile(availablePluginsJSON, file, 0755)
+	if err != nil {
+		fs.Logf(nil, "%s", err)
+	}
+	return nil
+}
+
+func (p *Plugins) removePlugin(name string) (err error) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	err = p.readFromFile()
+	if err != nil {
+		return err
+	}
+
+	_, ok := p.LoadedPlugins[name]
+	if !ok {
+		return fmt.Errorf("plugin %s not loaded", name)
+	}
+	delete(p.LoadedPlugins, name)
+
+	err = p.writeToFile()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// GetPluginByName returns the plugin object for the key (author/plugin-name)
+func (p *Plugins) GetPluginByName(name string) (out *PackageJSON, err error) {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	po, ok := p.LoadedPlugins[name]
+	if !ok {
+		return nil, fmt.Errorf("plugin %s not loaded", name)
+	}
+	return &po, nil
+
+}
+
+// getAuthorRepoBranchGithub gives author, repoName and branch from a github.com url
+//	url examples:
+//	https://github.com/rclone/rclone-webui-react/
+//	http://github.com/rclone/rclone-webui-react
+//	https://github.com/rclone/rclone-webui-react/tree/caman-js
+// 	github.com/rclone/rclone-webui-react
+//
+func getAuthorRepoBranchGithub(url string) (author string, repoName string, branch string, err error) {
+	repoURL := url
+	repoURL = strings.Replace(repoURL, "https://", "", 1)
+	repoURL = strings.Replace(repoURL, "http://", "", 1)
+
+	urlSplits := strings.Split(repoURL, "/")
+
+	if len(urlSplits) < 3 || len(urlSplits) > 5 || urlSplits[0] != "github.com" {
+		return "", "", "", fmt.Errorf("invalid github url: %s", url)
+	}
+
+	// get branch name
+	if len(urlSplits) == 5 && urlSplits[3] == "tree" {
+		return urlSplits[1], urlSplits[2], urlSplits[4], nil
+	}
+
+	return urlSplits[1], urlSplits[2], "master", nil
+}
+
+func filterPlugins(plugins *Plugins, compare func(packageJSON *PackageJSON) bool) map[string]PackageJSON {
+	output := map[string]PackageJSON{}
+
+	for key, val := range plugins.LoadedPlugins {
+		if compare(&val) {
+			output[key] = val
+		}
+	}
+
+	return output
+}
+
+// getDirectorForProxy is a helper function for reverse proxy of test plugins
+func getDirectorForProxy(origin *url.URL) func(req *http.Request) {
+	return func(req *http.Request) {
+		req.Header.Add("X-Forwarded-Host", req.Host)
+		req.Header.Add("X-Origin-Host", origin.Host)
+		req.URL.Scheme = "http"
+		req.URL.Host = origin.Host
+		req.URL.Path = origin.Path
+	}
+}
+
+// ServePluginOK checks the plugin url and uses reverse proxy to allow redirection for content not being served by rclone
+func ServePluginOK(w http.ResponseWriter, r *http.Request, pluginsMatchResult []string) (ok bool) {
+	testPlugin, err := loadedPlugins.GetPluginByName(fmt.Sprintf("%s/%s", pluginsMatchResult[1], pluginsMatchResult[2]))
+	if err != nil {
+		return false
+	}
+	if !testPlugin.Rclone.Test {
+		return false
+	}
+	origin, _ := url.Parse(fmt.Sprintf("%s/%s", testPlugin.TestURL, pluginsMatchResult[3]))
+
+	director := getDirectorForProxy(origin)
+
+	pluginsProxy.Director = director
+	pluginsProxy.ServeHTTP(w, r)
+	return true
+}
+
+var referrerPathReg = regexp.MustCompile("^(https?):\\/\\/(.+):([0-9]+)?\\/(.*)\\/?\\?(.*)$")
+
+// ServePluginWithReferrerOK check if redirectReferrer is set for the referred a plugin, if yes,
+// sends a redirect to actual url. This function is useful for plugins to refer to absolute paths when
+// the referrer in http.Request is set
+func ServePluginWithReferrerOK(w http.ResponseWriter, r *http.Request, path string) (ok bool) {
+	referrer := r.Referer()
+	referrerPathMatch := referrerPathReg.FindStringSubmatch(referrer)
+
+	if referrerPathMatch != nil && len(referrerPathMatch) > 3 {
+		referrerPluginMatch := PluginsMatch.FindStringSubmatch(referrerPathMatch[4])
+		if referrerPluginMatch != nil && len(referrerPluginMatch) > 2 {
+			pluginKey := fmt.Sprintf("%s/%s", referrerPluginMatch[1], referrerPluginMatch[2])
+			currentPlugin, err := loadedPlugins.GetPluginByName(pluginKey)
+			if err != nil {
+				return false
+			}
+			if currentPlugin.Rclone.RedirectReferrer {
+				path = fmt.Sprintf("/plugins/%s/%s/%s", referrerPluginMatch[1], referrerPluginMatch[2], path)
+
+				http.Redirect(w, r, path, http.StatusMovedPermanently)
+				return true
+			}
+		}
+	}
+	return false
+}
diff --git a/fs/rc/webgui/rc.go b/fs/rc/webgui/rc.go
new file mode 100644
index 000000000..8006cce48
--- /dev/null
+++ b/fs/rc/webgui/rc.go
@@ -0,0 +1,310 @@
+package webgui
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/rc"
+)
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "pluginsctl/listTestPlugins",
+		AuthRequired: true,
+		Fn:           rcListTestPlugins,
+		Title:        "Show currently loaded test plugins",
+		Help: `allows listing of test plugins with the rclone.test set to true in package.json of the plugin
+
+This takes no parameters and returns
+
+- loadedTestPlugins: list of currently available test plugins
+
+Eg
+
+    rclone rc pluginsctl/listTestPlugins
+`,
+	})
+}
+
+func rcListTestPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) {
+	return rc.Params{
+		"loadedTestPlugins": filterPlugins(loadedPlugins, func(json *PackageJSON) bool { return json.isTesting() }),
+	}, nil
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "pluginsctl/removeTestPlugin",
+		AuthRequired: true,
+		Fn:           rcRemoveTestPlugin,
+		Title:        "Remove  a test plugin",
+		Help: `This allows you to remove a plugin using it's name
+
+This takes the following parameters
+
+- name: name of the plugin in the format <author>/<plugin_name>
+
+Eg
+
+    rclone rc pluginsctl/removeTestPlugin name=rclone/rclone-webui-react
+`,
+	})
+}
+func rcRemoveTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
+	name, err := in.GetString("name")
+	if err != nil {
+		return nil, err
+	}
+	err = loadedPlugins.removePlugin(name)
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "pluginsctl/addPlugin",
+		AuthRequired: true,
+		Fn:           rcAddPlugin,
+		Title:        "Add a plugin using url",
+		Help: `used for adding a plugin to the webgui
+
+This takes the following parameters
+
+- url: http url of the github repo where the plugin is hosted (http://github.com/rclone/rclone-webui-react)
+
+Eg
+
+   rclone rc pluginsctl/addPlugin
+`,
+	})
+}
+
+func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
+	pluginURL, err := in.GetString("url")
+	if err != nil {
+		return nil, err
+	}
+
+	author, repoName, repoBranch, err := getAuthorRepoBranchGithub(pluginURL)
+	if err != nil {
+		return nil, err
+	}
+
+	branch, err := in.GetString("branch")
+	if err != nil || branch == "" {
+		branch = repoBranch
+	}
+
+	version, err := in.GetString("version")
+	if err != nil || version == "" {
+		version = "latest"
+	}
+
+	err = CreatePathIfNotExist(PluginsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// fetch and package.json
+	// https://raw.githubusercontent.com/rclone/rclone-webui-react/master/package.json
+
+	pluginID := fmt.Sprintf("%s/%s", author, repoName)
+
+	currentPluginPath := filepath.Join(PluginsPath, pluginID)
+
+	err = CreatePathIfNotExist(currentPluginPath)
+	if err != nil {
+		return nil, err
+	}
+
+	packageJSONUrl := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/package.json", author, repoName, branch)
+	packageJSONFilePath := filepath.Join(currentPluginPath, "package.json")
+	err = DownloadFile(packageJSONFilePath, packageJSONUrl)
+	if err != nil {
+		return nil, err
+	}
+	// register in plugins
+
+	// download release and save in plugins/<author>/repo-name/app
+	// https://api.github.com/repos/rclone/rclone-webui-react/releases/latest
+	releaseURL, tag, _, err := GetLatestReleaseURL(fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/%s", author, repoName, version))
+	if err != nil {
+		return nil, err
+	}
+	zipName := tag + ".zip"
+	zipPath := filepath.Join(currentPluginPath, zipName)
+
+	err = DownloadFile(zipPath, releaseURL)
+	if err != nil {
+		return nil, err
+	}
+
+	extractPath := filepath.Join(currentPluginPath, "app")
+
+	err = CreatePathIfNotExist(extractPath)
+	if err != nil {
+		return nil, err
+	}
+	err = os.RemoveAll(extractPath)
+	if err != nil {
+		fs.Logf(nil, "No previous downloads to remove")
+	}
+
+	fs.Logf(nil, "Unzipping plugin binary")
+
+	err = Unzip(zipPath, extractPath)
+	if err != nil {
+		return nil, err
+	}
+
+	err = loadedPlugins.addPlugin(pluginID, packageJSONFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "pluginsctl/listPlugins",
+		AuthRequired: true,
+		Fn:           rcGetPlugins,
+		Title:        "Get the list of currently loaded plugins",
+		Help: `This allows you to get the currently enabled plugins and their details.
+
+This takes no parameters and returns
+
+- loadedPlugins: list of current production plugins
+- testPlugins: list of temporarily loaded development plugins, usually running on a different server.
+
+Eg
+
+   rclone rc pluginsctl/listPlugins
+`,
+	})
+}
+
+func rcGetPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) {
+	err = loadedPlugins.readFromFile()
+	if err != nil {
+		return nil, err
+	}
+	return rc.Params{
+		"loadedPlugins":     filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { return !packageJSON.isTesting() }),
+		"loadedTestPlugins": filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool { return packageJSON.isTesting() }),
+	}, nil
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "pluginsctl/removePlugin",
+		AuthRequired: true,
+		Fn:           rcRemovePlugin,
+		Title:        "Remove a loaded plugin",
+		Help: `This allows you to remove a plugin using it's name
+
+This takes parameters
+
+- name: name of the plugin in the format <author>/<plugin_name>
+
+Eg
+
+   rclone rc pluginsctl/removePlugin name=rclone/video-plugin
+`,
+	})
+}
+
+func rcRemovePlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
+	name, err := in.GetString("name")
+	if err != nil {
+		return nil, err
+	}
+
+	err = loadedPlugins.removePlugin(name)
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "pluginsctl/getPluginsForType",
+		AuthRequired: true,
+		Fn:           rcGetPluginsForType,
+		Title:        "Get plugins with type criteria",
+		Help: `This shows all possible plugins by a mime type
+
+This takes the following parameters
+
+- type: supported mime type by a loaded plugin eg (video/mp4, audio/mp3)
+- pluginType: filter plugins based on their type eg (DASHBOARD, FILE_HANDLER, TERMINAL) 
+
+and returns
+
+- loadedPlugins: list of current production plugins
+- testPlugins: list of temporarily loaded development plugins, usually running on a different server.
+
+Eg
+
+   rclone rc pluginsctl/getPluginsForType type=video/mp4
+`,
+	})
+}
+
+func rcGetPluginsForType(_ context.Context, in rc.Params) (out rc.Params, err error) {
+	handlesType, err := in.GetString("type")
+	if err != nil {
+		handlesType = ""
+	}
+
+	pluginType, err := in.GetString("pluginType")
+	if err != nil {
+		pluginType = ""
+	}
+	var loadedPluginsResult map[string]PackageJSON
+
+	var loadedTestPluginsResult map[string]PackageJSON
+
+	if pluginType == "" || pluginType == "FileHandler" {
+
+		loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
+			for i := range packageJSON.Rclone.HandlesType {
+				if packageJSON.Rclone.HandlesType[i] == handlesType && !packageJSON.Rclone.Test {
+					return true
+				}
+			}
+			return false
+		})
+
+		loadedTestPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
+			for i := range packageJSON.Rclone.HandlesType {
+				if packageJSON.Rclone.HandlesType[i] == handlesType && packageJSON.Rclone.Test {
+					return true
+				}
+			}
+			return false
+		})
+	} else {
+		loadedPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
+			return packageJSON.Rclone.PluginType == pluginType && !packageJSON.isTesting()
+		})
+
+		loadedTestPluginsResult = filterPlugins(loadedPlugins, func(packageJSON *PackageJSON) bool {
+			return packageJSON.Rclone.PluginType == pluginType && packageJSON.isTesting()
+		})
+	}
+
+	return rc.Params{
+		"loadedPlugins":     loadedPluginsResult,
+		"loadedTestPlugins": loadedTestPluginsResult,
+	}, nil
+
+}
diff --git a/fs/rc/webgui.go b/fs/rc/webgui/webgui.go
similarity index 77%
rename from fs/rc/webgui.go
rename to fs/rc/webgui/webgui.go
index 9b09d5c14..7e73cabed 100644
--- a/fs/rc/webgui.go
+++ b/fs/rc/webgui/webgui.go
@@ -1,6 +1,6 @@
 // Define the Web GUI helpers
 
-package rc
+package webgui
 
 import (
 	"archive/zip"
@@ -15,21 +15,28 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/lib/errors"
 )
 
-// getLatestReleaseURL returns the latest release details of the rclone-webui-react
-func getLatestReleaseURL(fetchURL string) (string, string, int, error) {
+// GetLatestReleaseURL returns the latest release details of the rclone-webui-react
+func GetLatestReleaseURL(fetchURL string) (string, string, int, error) {
 	resp, err := http.Get(fetchURL)
 	if err != nil {
-		return "", "", 0, errors.New("Error getting latest release of rclone-webui")
+		return "", "", 0, errors.Wrap(err, "failed getting latest release of rclone-webui")
+	}
+	defer fs.CheckClose(resp.Body, &err)
+	if resp.StatusCode != http.StatusOK {
+		return "", "", 0, errors.Errorf("bad HTTP status %d (%s) when fetching %s", resp.StatusCode, resp.Status, fetchURL)
 	}
 	results := gitHubRequest{}
 	if err := json.NewDecoder(resp.Body).Decode(&results); err != nil {
-		return "", "", 0, errors.New("Could not decode results from http request")
+		return "", "", 0, errors.Wrap(err, "could not decode results from http request")
+	}
+	if len(results.Assets) < 1 {
+		return "", "", 0, errors.New("could not find an asset in the release. " +
+			"check if asset was successfully added in github release assets")
 	}
-
 	res := results.Assets[0].BrowserDownloadURL
 	tag := results.TagName
 	size := results.Assets[0].Size
@@ -56,7 +63,7 @@ func CheckAndDownloadWebGUIRelease(checkUpdate bool, forceUpdate bool, fetchURL
 	// TODO: Add hashing to check integrity of the previous update.
 	if !extractPathExist || checkUpdate || forceUpdate {
 		// Get the latest release details
-		WebUIURL, tag, size, err := getLatestReleaseURL(fetchURL)
+		WebUIURL, tag, size, err := GetLatestReleaseURL(fetchURL)
 		if err != nil {
 			return err
 		}
@@ -88,7 +95,7 @@ func CheckAndDownloadWebGUIRelease(checkUpdate bool, forceUpdate bool, fetchURL
 		fs.Logf(nil, "Downloading webgui binary. Please wait. [Size: %s, Path :  %s]\n", strconv.Itoa(size), zipPath)
 
 		// download the zip from latest url
-		err = downloadFile(zipPath, WebUIURL)
+		err = DownloadFile(zipPath, WebUIURL)
 		if err != nil {
 			return err
 		}
@@ -99,7 +106,7 @@ func CheckAndDownloadWebGUIRelease(checkUpdate bool, forceUpdate bool, fetchURL
 		}
 		fs.Logf(nil, "Unzipping webgui binary")
 
-		err = unzip(zipPath, extractPath)
+		err = Unzip(zipPath, extractPath)
 		if err != nil {
 			return err
 		}
@@ -120,15 +127,17 @@ func CheckAndDownloadWebGUIRelease(checkUpdate bool, forceUpdate bool, fetchURL
 	return nil
 }
 
-// downloadFile is a helper function to download a file from url to the filepath
-func downloadFile(filepath string, url string) error {
-
+// DownloadFile is a helper function to download a file from url to the filepath
+func DownloadFile(filepath string, url string) (err error) {
 	// Get the data
 	resp, err := http.Get(url)
 	if err != nil {
 		return err
 	}
 	defer fs.CheckClose(resp.Body, &err)
+	if resp.StatusCode != http.StatusOK {
+		return errors.Errorf("bad HTTP status %d (%s) when fetching %s", resp.StatusCode, resp.Status, url)
+	}
 
 	// Create the file
 	out, err := os.Create(filepath)
@@ -142,8 +151,8 @@ func downloadFile(filepath string, url string) error {
 	return err
 }
 
-// unzip is a helper function to unzip a file specified in src to path dest
-func unzip(src, dest string) (err error) {
+// Unzip is a helper function to Unzip a file specified in src to path dest
+func Unzip(src, dest string) (err error) {
 	dest = filepath.Clean(dest) + string(os.PathSeparator)
 
 	r, err := zip.OpenReader(src)
@@ -213,6 +222,22 @@ func exists(path string) (existence bool, stat os.FileInfo, err error) {
 	return false, stat, err
 }
 
+// CreatePathIfNotExist creates the path to a folder if it does not exist
+func CreatePathIfNotExist(path string) (err error) {
+	exists, stat, _ := exists(path)
+	if !exists {
+		if err := os.MkdirAll(path, 0755); err != nil {
+			return errors.New("Error creating : " + path)
+		}
+	}
+
+	if exists && !stat.IsDir() {
+		return errors.New("Path is a file instead of folder. Please check it " + path)
+	}
+
+	return nil
+}
+
 // gitHubRequest Maps the GitHub API request to structure
 type gitHubRequest struct {
 	URL string `json:"url"`
diff --git a/fs/version.go b/fs/version.go
index 5f1f636b4..0110ac8d3 100644
--- a/fs/version.go
+++ b/fs/version.go
@@ -1,4 +1,4 @@
 package fs
 
 // Version of rclone
-var Version = "v1.52.3-DEV"
+var Version = "v1.53.0-DEV"
diff --git a/fs/version/version.go b/fs/version/version.go
deleted file mode 100644
index a2accfe0b..000000000
--- a/fs/version/version.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package version
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-
-	"github.com/pkg/errors"
-	"github.com/rclone/rclone/fs"
-)
-
-// Version represents a parsed rclone version number
-type Version []int
-
-var parseVersion = regexp.MustCompile(`^(?:rclone )?v(\d+)\.(\d+)(?:\.(\d+))?(?:-(\d+)(?:-(g[\wβ-]+))?)?$`)
-
-// New parses a version number from a string
-//
-// This will be returned with up to 4 elements for major, minor,
-// patch, subpatch release.
-//
-// If the version number represents a compiled from git version
-// number, then it will be returned as major, minor, 999, 999
-func New(in string) (v Version, err error) {
-	isGit := strings.HasSuffix(in, "-DEV")
-	if isGit {
-		in = in[:len(in)-4]
-	}
-	r := parseVersion.FindStringSubmatch(in)
-	if r == nil {
-		return v, errors.Errorf("failed to match version string %q", in)
-	}
-	atoi := func(s string) int {
-		i, err := strconv.Atoi(s)
-		if err != nil {
-			fs.Errorf(nil, "Failed to parse %q as int from %q: %v", s, in, err)
-		}
-		return i
-	}
-	v = Version{
-		atoi(r[1]), // major
-		atoi(r[2]), // minor
-	}
-	if r[3] != "" {
-		v = append(v, atoi(r[3])) // patch
-	} else if r[4] != "" {
-		v = append(v, 0) // patch
-	}
-	if r[4] != "" {
-		v = append(v, atoi(r[4])) // dev
-	}
-	if isGit {
-		v = append(v, 999, 999)
-	}
-	return v, nil
-}
-
-// String converts v to a string
-func (v Version) String() string {
-	var out []string
-	for _, vv := range v {
-		out = append(out, fmt.Sprint(vv))
-	}
-	return strings.Join(out, ".")
-}
-
-// Cmp compares two versions returning >0, <0 or 0
-func (v Version) Cmp(o Version) (d int) {
-	n := len(v)
-	if n > len(o) {
-		n = len(o)
-	}
-	for i := 0; i < n; i++ {
-		d = v[i] - o[i]
-		if d != 0 {
-			return d
-		}
-	}
-	return len(v) - len(o)
-}
-
-// IsGit returns true if the current version was compiled from git
-func (v Version) IsGit() bool {
-	return len(v) >= 4 && v[2] == 999 && v[3] == 999
-}
diff --git a/go.mod b/go.mod
index d9854ccd3..a8a257675 100644
--- a/go.mod
+++ b/go.mod
@@ -25,6 +25,7 @@ require (
 	github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e
 	github.com/jzelinskie/whirlpool v0.0.0-20170603002051-c19460b8caa6
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/klauspost/compress v1.10.11
 	github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348
 	github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a
 	github.com/mattn/go-colorable v0.1.7
@@ -62,12 +63,12 @@ require (
 	golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666
 	golang.org/x/text v0.3.3
 	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1
+	golang.org/x/tools v0.0.0-20200820180210-c8f393745106 // indirect
 	google.golang.org/api v0.28.0
 	google.golang.org/genproto v0.0.0-20200626011028-ee7919e894b5 // indirect
 	google.golang.org/grpc v1.30.0 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
-	storj.io/common v0.0.0-20200624215549-bf610d22d466 // indirect
-	storj.io/uplink v1.1.2
+	storj.io/uplink v1.2.0
 )
diff --git a/go.sum b/go.sum
index 2b4c0f7d9..5319ab629 100644
--- a/go.sum
+++ b/go.sum
@@ -87,10 +87,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/billziss-gh/cgofuse v1.3.0 h1:mFj8XQg/vvxMFywNy1F7IqFYcMeBqceYTh1+iUhpsk8=
-github.com/billziss-gh/cgofuse v1.3.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
-github.com/billziss-gh/cgofuse v1.3.1-0.20200703171401-45df47debffe h1:AXqxouOOD7FQuoVfZubWmMyHzOrrSGZbdh9o6PCtfKM=
-github.com/billziss-gh/cgofuse v1.3.1-0.20200703171401-45df47debffe/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
 github.com/billziss-gh/cgofuse v1.4.0 h1:kju2jDmdNuDDCrxPob2ggmZr5Mj/odCjU1Y8kx0Th9E=
 github.com/billziss-gh/cgofuse v1.4.0/go.mod h1:LJjoaUojlVjgo5GQoEJTcJNqZJeRU0nCR84CyxKt2YM=
 github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
@@ -107,8 +103,6 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa
 github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
-github.com/calebcase/tmpfile v1.0.1 h1:vD8FSrbsbexhep39/6mvtbIHS3GzIRqiprDNCF6QqSk=
-github.com/calebcase/tmpfile v1.0.1/go.mod h1:iErLeG/iqJr8LaQ/gYRv4GXdqssi3jg4iSzvrA06/lw=
 github.com/calebcase/tmpfile v1.0.2-0.20200602150926-3af473ef8439/go.mod h1:iErLeG/iqJr8LaQ/gYRv4GXdqssi3jg4iSzvrA06/lw=
 github.com/calebcase/tmpfile v1.0.2 h1:1AGuhKiUu4J6wxz6lxuF6ck3f8G2kaV6KSEny0RGCig=
 github.com/calebcase/tmpfile v1.0.2/go.mod h1:iErLeG/iqJr8LaQ/gYRv4GXdqssi3jg4iSzvrA06/lw=
@@ -265,6 +259,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk=
+github.com/klauspost/compress v1.10.11/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -431,6 +427,7 @@ github.com/youmark/pkcs8 v0.0.0-20200520070018-fad002e585ce h1:F5MEHq8k6JiE10MNY
 github.com/youmark/pkcs8 v0.0.0-20200520070018-fad002e585ce/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8=
 github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4=
 github.com/zeebo/admission/v3 v3.0.1/go.mod h1:BP3isIv9qa2A7ugEratNq1dnl2oZRXaQUGdU7WXKtbw=
@@ -479,7 +476,6 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
@@ -548,6 +544,7 @@ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -588,7 +585,6 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200107144601-ef85f5a75ddf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -660,6 +656,8 @@ golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa h1:mMXQKlWCw9mIWgVLLfiycDZjMHMMYqiuakI4E/l2xcA=
 golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200820180210-c8f393745106 h1:42Zs/g7pjhSIE/wiAuKcp8zp20zv7W2diNU6arpshOA=
+golang.org/x/tools v0.0.0-20200820180210-c8f393745106/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
@@ -777,13 +775,9 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-storj.io/common v0.0.0-20200611114417-9a3d012fdb62 h1:y8vGNQ0HjtD79G8MfCwbs6hct40tSBoDaOnsxWOZpU4=
-storj.io/common v0.0.0-20200611114417-9a3d012fdb62/go.mod h1:6S6Ub92/BB+ofU7hbyPcm96b4Q1ayyN0HLog+3u+wGc=
-storj.io/common v0.0.0-20200624215549-bf610d22d466 h1:XQM2PQMJMoEaRluJOYfVONi/IclVhJgfEEPDVLAt5MU=
-storj.io/common v0.0.0-20200624215549-bf610d22d466/go.mod h1:vMAnlNbkgW6i+w/OT1h4X8w6TajOHWAT+SvFHUFCpq0=
-storj.io/drpc v0.0.12 h1:4ei1M4cnWlYxcQheX0Dg4+c12zCD+oJqfweVQVWarsA=
-storj.io/drpc v0.0.12/go.mod h1:82nfl+6YwRwF6UG31cEWWUqv/FaKvP5SGqUvoqTxCMA=
-storj.io/drpc v0.0.13 h1:EDR3WiwVcIHtg+8M5vqBFmUAuJvmM2erVHIfqPPSAoc=
-storj.io/drpc v0.0.13/go.mod h1:82nfl+6YwRwF6UG31cEWWUqv/FaKvP5SGqUvoqTxCMA=
-storj.io/uplink v1.1.2 h1:r0EyoEDlAvqWd6SZDG10w0k2+CjSMi4wAq5J1Sw8y9Y=
-storj.io/uplink v1.1.2/go.mod h1:UkdYN/dfSgv+d8fBUoZTrX2oLdj9gzX6Q7tp3CojgKA=
+storj.io/common v0.0.0-20200729140050-4c1ddac6fa63 h1:BkRvlginTJGi0yAkpN+4ZKm2YpG63bDSDFLQtXYxxdg=
+storj.io/common v0.0.0-20200729140050-4c1ddac6fa63/go.mod h1:ILr54ISCqCQ6MmIwT7eaR/fEGrBfgfxiPt8nmpWqnUM=
+storj.io/drpc v0.0.14 h1:GCBdymTt1BRw4oHmmUZZlxYXLVRxxYj6x3Ivide2J+I=
+storj.io/drpc v0.0.14/go.mod h1:82nfl+6YwRwF6UG31cEWWUqv/FaKvP5SGqUvoqTxCMA=
+storj.io/uplink v1.2.0 h1:7gGfkTv7zT9ivSCMqu7QirUMdHVaeBnlZgqgWhRKEd4=
+storj.io/uplink v1.2.0/go.mod h1:U7VFTdoZiBgmijKdvB5dUfi6V5Ew9PAHhUVyy8rJWBs=
diff --git a/lib/encoder/filename/decode.go b/lib/encoder/filename/decode.go
new file mode 100644
index 000000000..75f87ccd5
--- /dev/null
+++ b/lib/encoder/filename/decode.go
@@ -0,0 +1,84 @@
+package filename
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"errors"
+	"sync"
+
+	"github.com/klauspost/compress/huff0"
+)
+
+// ErrCorrupted is returned if a provided encoded filename cannot be decoded.
+var ErrCorrupted = errors.New("file name corrupt")
+
+// ErrUnsupported is returned if a provided encoding may come from a future version or the file name is corrupt.
+var ErrUnsupported = errors.New("file name possibly generated by future version of rclone")
+
+// Custom decoder for tableCustom types. Stateful, so must have lock.
+var customDec huff0.Scratch
+var customDecMu sync.Mutex
+
+// Decode an encoded string.
+func Decode(s string) (string, error) {
+	if len(s) < 1 {
+		return "", ErrCorrupted
+	}
+	table := decodeMap[s[0]]
+	if table == 0 {
+		return "", ErrCorrupted
+	}
+	table--
+	s = s[1:]
+
+	data := make([]byte, base64.URLEncoding.DecodedLen(len(s)))
+	n, err := base64.URLEncoding.Decode(data, ([]byte)(s))
+	if err != nil || n < 0 {
+		return "", ErrCorrupted
+	}
+	data = data[:n]
+
+	switch table {
+	case tableUncompressed:
+		return string(data), nil
+	case tableReserved:
+		return "", ErrUnsupported
+	case tableRLE:
+		if len(data) < 2 {
+			return "", ErrCorrupted
+		}
+		n, used := binary.Uvarint(data[:len(data)-1])
+		if used <= 0 || n > maxLength {
+			return "", ErrCorrupted
+		}
+		return string(bytes.Repeat(data[len(data)-1:], int(n))), nil
+	case tableCustom:
+		customDecMu.Lock()
+		defer customDecMu.Unlock()
+		_, data, err := huff0.ReadTable(data, &customDec)
+		if err != nil {
+			return "", ErrCorrupted
+		}
+		customDec.MaxDecodedSize = maxLength
+		decoded, err := customDec.Decompress1X(data)
+		if err != nil {
+			return "", ErrCorrupted
+		}
+		return string(decoded), nil
+	default:
+		if table >= byte(len(decTables)) {
+			return "", ErrCorrupted
+		}
+		dec := decTables[table]
+		if dec == nil {
+			return "", ErrUnsupported
+		}
+		var dst [maxLength]byte
+		name, err := dec.Decompress1X(dst[:0], data)
+		if err != nil {
+			return "", ErrCorrupted
+		}
+		return string(name), nil
+	}
+}
diff --git a/lib/encoder/filename/encode.go b/lib/encoder/filename/encode.go
new file mode 100644
index 000000000..555bf6b91
--- /dev/null
+++ b/lib/encoder/filename/encode.go
@@ -0,0 +1,60 @@
+package filename
+
+import (
+	"encoding/base64"
+	"encoding/binary"
+
+	"github.com/klauspost/compress/huff0"
+)
+
+// Encode will encode the string and return a base64 (url) compatible version of it.
+// Calling Decode with the returned string should always succeed.
+// It is not a requirement that the input string is valid utf-8.
+func Encode(s string) string {
+	initCoders()
+	bestSize := len(s)
+	bestTable := tableUncompressed
+	org := []byte(s)
+	bestOut := []byte(s)
+
+	// Try all tables and choose the best
+	for i, enc := range encTables[:] {
+		if len(org) <= 1 || len(org) > maxLength {
+			// Use the uncompressed
+			break
+		}
+		if enc == nil {
+			continue
+		}
+		// Try to encode using table.
+		err := func() error {
+			encTableLocks[i].Lock()
+			defer encTableLocks[i].Unlock()
+			out, _, err := huff0.Compress1X(org, enc)
+			if err != nil {
+				return err
+			}
+			if len(out) < bestSize {
+				bestOut = bestOut[:len(out)]
+				bestTable = i
+				bestSize = len(out)
+				copy(bestOut, out)
+			}
+			return nil
+		}()
+		// If input is a single byte repeated store as RLE or save uncompressed.
+		if err == huff0.ErrUseRLE {
+			if len(org) > 2 {
+				// Encode as one byte repeated since it will be smaller than uncompressed.
+				n := binary.PutUvarint(bestOut, uint64(len(org)))
+				bestOut = bestOut[:n+1]
+				bestOut[n] = org[0]
+				bestSize = n + 1
+				bestTable = tableRLE
+			}
+			break
+		}
+	}
+
+	return string(encodeURL[bestTable]) + base64.URLEncoding.EncodeToString(bestOut)
+}
diff --git a/lib/encoder/filename/fuzz.go b/lib/encoder/filename/fuzz.go
new file mode 100644
index 000000000..73468ee71
--- /dev/null
+++ b/lib/encoder/filename/fuzz.go
@@ -0,0 +1,33 @@
+//+build gofuzz
+
+package filename
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// Run like:
+// go-fuzz-build -o=fuzz-build.zip -func=Fuzz . && go-fuzz -minimize=5s -bin=fuzz-build.zip -workdir=testdata/corpus -procs=24
+
+// Fuzz test the provided input.
+func Fuzz(data []byte) int {
+	// First try to decode as is.
+	// We don't care about the result, it just shouldn't crash.
+	Decode(string(data))
+
+	// Now encode
+	enc := Encode(string(data))
+
+	// And decoded must match
+	decoded, err := Decode(enc)
+	if err != nil {
+		panic(fmt.Sprintf("error decoding %q, input %q: %v", enc, string(data), err))
+	}
+	if !bytes.Equal(data, []byte(decoded)) {
+		panic(fmt.Sprintf("decode mismatch, encoded: %q, org: %q, got: %q", enc, string(data), decoded))
+	}
+
+	// Everything is good.
+	return 1
+}
diff --git a/lib/encoder/filename/gentable.go b/lib/encoder/filename/gentable.go
new file mode 100644
index 000000000..4acb61707
--- /dev/null
+++ b/lib/encoder/filename/gentable.go
@@ -0,0 +1,93 @@
+//+build ignore
+
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"math"
+
+	"github.com/klauspost/compress"
+	"github.com/klauspost/compress/huff0"
+)
+
+// Replace/add histogram data and execute go run gentable.go
+
+func main() {
+	// Allow non-represented characters.
+	const omitUnused = false
+
+	histogram := [256]uint64{
+		// ncw home directory
+		//0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19442, 760, 0, 349, 570, 1520, 199, 76, 685, 654, 0, 40377, 1605, 395132, 935270, 0, 1156377, 887730, 811737, 712241, 693240, 689139, 675964, 656417, 666577, 657413, 532, 24, 0, 145, 0, 3, 946, 44932, 37362, 46126, 36752, 76346, 19338, 47457, 14288, 38163, 4350, 7867, 36541, 65011, 30255, 26792, 22097, 1803, 39191, 61965, 76585, 11887, 12896, 5931, 1935, 1731, 1385, 1279, 9, 1278, 1, 420185, 0, 1146359, 746359, 968896, 868703, 1393640, 745019, 354147, 159462, 483979, 169092, 75937, 385858, 322166, 466635, 571268, 447132, 13792, 446484, 736844, 732675, 170232, 112983, 63184, 142357, 173945, 21521, 250, 0, 250, 4140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 10, 0, 5, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		//Images:
+		//0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 0, 0, 8, 7, 3, 3, 0, 29, 53, 247265, 83587, 0, 265952, 233552, 229781, 71156, 78374, 65141, 46152, 43767, 55603, 39411, 0, 0, 0, 0, 0, 88, 84, 141, 70, 222, 191, 51, 52, 101, 60, 53, 23, 17, 49, 93, 53, 17, 92, 0, 158, 109, 41, 19, 43, 28, 10, 5, 1, 0, 0, 0, 0, 879, 0, 3415, 6770, 39823, 3566, 2491, 964, 42115, 825, 5178, 40755, 483, 1290, 3294, 1720, 6309, 42983, 10, 37739, 3454, 7028, 5077, 854, 227, 1259, 767, 218, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		// Google Drive:
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 7, 0, 0, 0, 7, 1, 1, 0, 2, 1, 506, 706, 0, 3903, 3552, 3694, 3338, 3262, 3257, 3222, 3249, 3325, 3261, 5, 0, 0, 1, 0, 0, 0, 48, 31, 61, 53, 46, 17, 17, 34, 32, 9, 22, 17, 31, 27, 19, 52, 5, 46, 84, 38, 14, 5, 19, 2, 2, 0, 8, 0, 8, 0, 180, 0, 5847, 3282, 3729, 3695, 3842, 3356, 316, 139, 487, 117, 95, 476, 289, 428, 609, 467, 5, 446, 592, 955, 130, 112, 57, 390, 168, 14, 0, 2, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	}
+
+	// Override with equally distributed characters
+	if false {
+		histogram = [256]uint64{}
+		var chars string
+		// base c64
+		chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+		// hex
+		//chars = "0123456789abcdef"
+		for _, v := range []byte(chars) {
+			histogram[v] = 1
+		}
+	}
+
+	// Sum up distributions
+	var total uint64
+	for _, v := range histogram[:] {
+		total += v
+	}
+
+	// Scale the distribution to approx this size.
+	const scale = 100 << 10
+	var tmp []byte
+	for i, v := range histogram[:] {
+		if v == 0 && omitUnused {
+			continue
+		}
+		nf := float64(v) / float64(total) * scale
+		if nf < 1 {
+			nf = 1
+		}
+		t2 := make([]byte, int(math.Ceil(nf)))
+		for j := range t2 {
+			t2[j] = byte(i)
+		}
+		tmp = append(tmp, t2...)
+	}
+
+	var s huff0.Scratch
+	s.Reuse = huff0.ReusePolicyNone
+	_, _, err := huff0.Compress1X(tmp, &s)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println("table:", base64.URLEncoding.EncodeToString(s.OutTable))
+
+	// Encode without ones:
+	s.Reuse = huff0.ReusePolicyPrefer
+	tmp = tmp[:0]
+	for i, v := range histogram[:] {
+		nf := float64(v) / float64(total) * scale
+		t2 := make([]byte, int(math.Ceil(nf)))
+		for j := range t2 {
+			t2[j] = byte(i)
+		}
+		tmp = append(tmp, t2...)
+	}
+	_, _, err = huff0.Compress1X(tmp, &s)
+	fmt.Println("sample", len(tmp), "byte, compressed size:", len(s.OutData))
+	fmt.Println("Shannon limit:", compress.ShannonEntropyBits(tmp)/8, "bytes")
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("avg size: 1 -> %.02f", float64(len(s.OutData))/float64(len(tmp)))
+}
diff --git a/lib/encoder/filename/init.go b/lib/encoder/filename/init.go
new file mode 100644
index 000000000..1a7ae41ac
--- /dev/null
+++ b/lib/encoder/filename/init.go
@@ -0,0 +1,89 @@
+package filename
+
+import (
+	"encoding/base64"
+	"sync"
+
+	"github.com/klauspost/compress/huff0"
+)
+
+// encodeURL is base64 url encoding values.
+const encodeURL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+
+// decodeMap will return x = decodeMap[encodeURL[byte(x)]] - 1 if x >= 0 and x < 64, otherwise -1 is returned.
+var decodeMap [256]byte
+
+// maxLength is the maximum length that will be attempted to be compressed.
+const maxLength = 256
+
+var (
+	initOnce sync.Once // Used to control init of tables.
+
+	encTables     [64]*huff0.Scratch // Encoders.
+	encTableLocks [64]sync.Mutex     // Temporary locks for encoders since they are stateful.
+	decTables     [64]*huff0.Decoder // Stateless decoders.
+)
+
+const (
+	tableUncompressed = 0
+	tableRLE          = 61
+	tableCustom       = 62
+	tableReserved     = 63
+)
+
+// predefined tables as base64 URL encoded string.
+var tablesData = [64]string{
+	// Uncompressed
+	tableUncompressed: "",
+	// ncw home directory
+	1: "MRDIEtAAMAzDMAzDSjX_ybu0w97bb-L3b2mR-rUl5LXW3lZII43kIDMzM1NXu3okgQs=",
+	// ncw images
+	2: "IhDIAEAA______-Pou_4Sf5z-uS-39MVWjullFLKM7EBECs=",
+	// ncw Google Drive:
+	3: "JxDQAIIBMDMzMwOzbv7nJJCyd_m_9D2llCarnQX33nvvlFKEhUxAAQ==",
+	// Hex
+	4: "ExDoSTD___-tfXfhJ0hKSkryTxU=",
+	// Base64
+	5: "JRDIcQf_______8PgIiIiIgINkggARHlkQwSSCCBxHFYINHdfXI=",
+
+	// Special tables:
+	// Compressed data has its own table.
+	tableCustom: "",
+	// Reserved for extension.
+	tableReserved: "",
+}
+
+func initCoders() {
+	initOnce.Do(func() {
+		// Init base 64 decoder.
+		for i, v := range encodeURL {
+			decodeMap[v] = byte(i) + 1
+		}
+
+		// Initialize encoders and decoders.
+		for i, dataString := range tablesData {
+			if len(dataString) == 0 {
+				continue
+			}
+			data, err := base64.URLEncoding.DecodeString(dataString)
+			if err != nil {
+				panic(err)
+			}
+			s, _, err := huff0.ReadTable(data, nil)
+			if err != nil {
+				panic(err)
+			}
+
+			// We want to save at least len(in) >> 5
+			s.WantLogLess = 5
+			s.Reuse = huff0.ReusePolicyMust
+			encTables[i] = s
+			decTables[i] = s.Decoder()
+		}
+		// Add custom table type.
+		var s huff0.Scratch
+		s.Reuse = huff0.ReusePolicyNone
+		encTables[tableCustom] = &s
+		decTables[tableCustom] = nil
+	})
+}
diff --git a/vfs/file.go b/vfs/file.go
index 5a0cd5ca8..eb1103cd3 100644
--- a/vfs/file.go
+++ b/vfs/file.go
@@ -186,6 +186,7 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
 
 		f.mu.RLock()
 		o := f.o
+		d := f.d
 		f.mu.RUnlock()
 		var newObject fs.Object
 		// if o is nil then are writing the file so no need to rename the object
@@ -210,8 +211,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
 			}
 		}
 		// Rename in the cache
-		if f.d.vfs.cache != nil {
-			if err := f.d.vfs.cache.Rename(oldPath, newPath, newObject); err != nil {
+		if d.vfs.cache != nil && d.vfs.cache.Exists(oldPath) {
+			if err := d.vfs.cache.Rename(oldPath, newPath, newObject); err != nil {
 				fs.Infof(f.Path(), "File.Rename failed in Cache: %v", err)
 			}
 		}
@@ -241,7 +242,7 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
 	CacheMode := d.vfs.Opt.CacheMode
 	if writing &&
 		(CacheMode < vfscommon.CacheModeMinimal ||
-			(CacheMode == vfscommon.CacheModeMinimal && !f.d.vfs.cache.Exists(oldPath))) {
+			(CacheMode == vfscommon.CacheModeMinimal && !destDir.vfs.cache.Exists(oldPath))) {
 		fs.Debugf(oldPath, "File is currently open, delaying rename %p", f)
 		f.mu.Lock()
 		f.pendingRenameFun = renameCall
@@ -342,14 +343,19 @@ func (f *File) Size() int64 {
 
 // SetModTime sets the modtime for the file
 func (f *File) SetModTime(modTime time.Time) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
 	if f.d.vfs.Opt.ReadOnly {
 		return EROFS
 	}
-	f.mu.Lock()
-	defer f.mu.Unlock()
 
 	f.pendingModTime = modTime
 
+	// set the time of the file in the cache
+	if f.d.vfs.cache != nil && f.d.vfs.cache.Exists(f._path()) {
+		f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime)
+	}
+
 	// Only update the ModTime when there are no writers, setObject will do it
 	if !f._writingInProgress() {
 		return f._applyPendingModTime()
@@ -365,7 +371,6 @@ func (f *File) _applyPendingModTime() error {
 	if f.pendingModTime.IsZero() {
 		return nil
 	}
-
 	defer func() { f.pendingModTime = time.Time{} }()
 
 	if f.o == nil {
@@ -376,19 +381,14 @@ func (f *File) _applyPendingModTime() error {
 	err := f.o.SetModTime(context.TODO(), f.pendingModTime)
 	switch err {
 	case nil:
-		fs.Debugf(f._path(), "File._applyPendingModTime OK")
+		fs.Debugf(f.o, "Applied pending mod time %v OK", f.pendingModTime)
 	case fs.ErrorCantSetModTime, fs.ErrorCantSetModTimeWithoutDelete:
 		// do nothing, in order to not break "touch somefile" if it exists already
 	default:
-		fs.Debugf(f._path(), "File._applyPendingModTime error: %v", err)
+		fs.Errorf(f.o, "Failed to apply pending mod time %v: %v", f.pendingModTime, err)
 		return err
 	}
 
-	// set the time of the file in the cache
-	if f.d.vfs.cache != nil {
-		f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime)
-	}
-
 	return nil
 }
 
@@ -408,10 +408,11 @@ func (f *File) setObject(o fs.Object) {
 	f.mu.Lock()
 	f.o = o
 	_ = f._applyPendingModTime()
+	d := f.d
 	f.mu.Unlock()
 
 	// Release File.mu before calling Dir method
-	f.d.addObject(f)
+	d.addObject(f)
 }
 
 // Update the object but don't update the directory cache - for use by
@@ -535,7 +536,7 @@ func (f *File) Remove() (err error) {
 
 	// Remove the object from the cache
 	wasWriting := false
-	if d.vfs.cache != nil {
+	if d.vfs.cache != nil && d.vfs.cache.Exists(f.Path()) {
 		wasWriting = d.vfs.cache.Remove(f.Path())
 	}
 
diff --git a/vfs/read_write.go b/vfs/read_write.go
index 790c43e1f..7fa04a7d6 100644
--- a/vfs/read_write.go
+++ b/vfs/read_write.go
@@ -260,7 +260,9 @@ func (fh *RWFileHandle) _readAt(b []byte, off int64, release bool) (n int, err e
 		// Do the writing with fh.mu unlocked
 		fh.mu.Unlock()
 	}
+
 	n, err = fh.item.ReadAt(b, off)
+
 	if release {
 		fh.mu.Lock()
 	}
diff --git a/vfs/vfs.go b/vfs/vfs.go
index 58d725648..d2d1d9d2b 100644
--- a/vfs/vfs.go
+++ b/vfs/vfs.go
@@ -216,7 +216,8 @@ func New(f fs.Fs, opt *vfscommon.Options) *VFS {
 	vfs.root = newDir(vfs, f, nil, fsDir)
 
 	// Start polling function
-	if do := vfs.f.Features().ChangeNotify; do != nil {
+	features := vfs.f.Features()
+	if do := features.ChangeNotify; do != nil {
 		vfs.pollChan = make(chan time.Duration)
 		do(context.TODO(), vfs.root.changeNotify, vfs.pollChan)
 		vfs.pollChan <- vfs.Opt.PollInterval
@@ -224,12 +225,17 @@ func New(f fs.Fs, opt *vfscommon.Options) *VFS {
 		fs.Infof(f, "poll-interval is not supported by this remote")
 	}
 
+	// Warn if can't stream
+	if !vfs.Opt.ReadOnly && vfs.Opt.CacheMode < vfscommon.CacheModeWrites && features.PutStream == nil {
+		fs.Logf(f, "--vfs-cache-mode writes or full is recommended for this remote as it can't stream")
+	}
+
 	vfs.SetCacheMode(vfs.Opt.CacheMode)
 
 	// Pin the Fs into the cache so that when we use cache.NewFs
 	// with the same remote string we get this one. The Pin is
-	// removed by Shutdown
-	cache.Pin(f)
+	// removed when the vfs is finalized
+	cache.PinUntilFinalized(f, vfs)
 
 	return vfs
 }
@@ -287,9 +293,6 @@ func (vfs *VFS) Shutdown() {
 		return
 	}
 
-	// Unpin the Fs from the cache
-	cache.Unpin(vfs.f)
-
 	// Remove from active cache
 	activeMu.Lock()
 	configName := fs.ConfigString(vfs.f)
diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go
index a6cf6eb45..eef969361 100644
--- a/vfs/vfscache/cache.go
+++ b/vfs/vfscache/cache.go
@@ -17,6 +17,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	fscache "github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/file"
@@ -47,9 +48,16 @@ type Cache struct {
 	writeback  *writeback.WriteBack // holds Items for writeback
 	avFn       AddVirtualFn         // if set, can be called to add dir entries
 
-	mu   sync.Mutex       // protects the following variables
-	item map[string]*Item // files/directories in the cache
-	used int64            // total size of files in the cache
+	mu            sync.Mutex       // protects the following variables
+	cond          *sync.Cond       // cond lock for synchronous cache cleaning
+	item          map[string]*Item // files/directories in the cache
+	errItems      map[string]error // items in error state
+	used          int64            // total size of files in the cache
+	outOfSpace    bool             // out of space
+	cleanerKicked bool             // some thread kicked the cleaner upon out of space
+	kickerMu      sync.Mutex       // mutex for clearnerKicked
+	kick          chan struct{}    // channel for kicking clear to start
+
 }
 
 // AddVirtualFn if registered by the WithAddVirtual method, can be
@@ -96,6 +104,7 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVir
 		root:       root,
 		metaRoot:   metaRoot,
 		item:       make(map[string]*Item),
+		errItems:   make(map[string]error),
 		hashType:   hashType,
 		hashOption: hashOption,
 		writeback:  writeback.New(ctx, opt),
@@ -117,6 +126,10 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVir
 	// Remove any empty directories
 	c.purgeEmptyDirs()
 
+	// Create a channel for cleaner to be kicked upon out of space con
+	c.kick = make(chan struct{}, 1)
+	c.cond = sync.NewCond(&c.mu)
+
 	go c.cleaner(ctx)
 
 	return c, nil
@@ -401,29 +414,126 @@ func (c *Cache) reload(ctx context.Context) error {
 	return nil
 }
 
-// purgeOld gets rid of any files that are over age
-func (c *Cache) purgeOld(maxAge time.Duration) {
-	c._purgeOld(maxAge, func(item *Item) {
-		item.remove("too old")
-	})
+// KickCleaner kicks cache cleaner upon out of space situation
+func (c *Cache) KickCleaner() {
+	/* Use a separate kicker mutex for the kick to go through without waiting for the
+	   cache mutex to avoid letting a thread kick again after the clearer just
+	   finished cleaning and unlock the cache mutex. */
+	fs.Debugf(nil, "vfs cache: at the beginning of KickCleaner")
+	c.kickerMu.Lock()
+	if !c.cleanerKicked {
+		c.cleanerKicked = true
+		fs.Debugf(nil, "vfs cache: in KickCleaner, ready to lock cache mutex")
+		c.mu.Lock()
+		c.outOfSpace = true
+		fs.Logf(nil, "vfs cache: in KickCleaner, ready to kick cleaner")
+		c.kick <- struct{}{}
+		c.mu.Unlock()
+	}
+	c.kickerMu.Unlock()
+
+	c.mu.Lock()
+	for c.outOfSpace == true {
+		fs.Debugf(nil, "vfs cache: in KickCleaner, looping on c.outOfSpace")
+		c.cond.Wait()
+	}
+	fs.Debugf(nil, "vfs cache: in KickCleaner, leaving c.outOfSpace loop")
+	c.mu.Unlock()
 }
 
-func (c *Cache) _purgeOld(maxAge time.Duration, remove func(item *Item)) {
+// removeNotInUse removes items not in use with a possible maxAge cutoff
+// called with cache mutex locked and up-to-date c.used (as we update it directly here)
+func (c *Cache) removeNotInUse(item *Item, maxAge time.Duration, emptyOnly bool) {
+	removed, spaceFreed := item.RemoveNotInUse(maxAge, emptyOnly)
+	// The item space might be freed even if we get an error after the cache file is removed
+	// The item will not be removed or reset the cache data is dirty (DataDirty)
+	c.used -= spaceFreed
+	if removed {
+		fs.Infof(nil, "vfs cache RemoveNotInUse (maxAge=%d, emptyOnly=%v): item %s was removed, freed %d bytes", maxAge, emptyOnly, item.GetName(), spaceFreed)
+		// Remove the entry
+		delete(c.item, item.name)
+	} else {
+		fs.Infof(nil, "vfs cache RemoveNotInUse (maxAge=%d, emptyOnly=%v): item %s not removed, freed %d bytes", maxAge, emptyOnly, item.GetName(), spaceFreed)
+	}
+	return
+}
+
+// Retry failed resets during purgeClean()
+func (c *Cache) retryFailedResets() {
+	// Some items may have failed to reset becasue there was not enough space
+	// for saving the cache item's metadata.  Redo the Reset()'s here now that
+	// we may have some available space.
+	if len(c.errItems) != 0 {
+		fs.Debugf(nil, "vfs cache reset: before redoing reset errItems = %v", c.errItems)
+		for itemName := range c.errItems {
+			_, _, err := c.item[itemName].Reset()
+			if err == nil || !fserrors.IsErrNoSpace(err) {
+				// TODO: not trying to handle non-ENOSPC errors yet
+				delete(c.errItems, itemName)
+			}
+		}
+		fs.Debugf(nil, "vfs cache reset: after redoing reset errItems = %v", c.errItems)
+	}
+}
+
+func (c *Cache) purgeClean(quota int64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	cutoff := time.Now().Add(-maxAge)
-	for name, item := range c.item {
-		if !item.inUse() {
-			// If not locked and access time too long ago - delete the file
-			dt := item.getATime().Sub(cutoff)
-			// fs.Debugf(name, "atime=%v cutoff=%v, dt=%v", item.info.ATime, cutoff, dt)
-			if dt < 0 {
-				remove(item)
-				// Remove the entry
-				delete(c.item, name)
-			}
+
+	var items Items
+
+	if quota <= 0 || c.used < quota {
+		return
+	}
+
+	// Make a slice of clean cache files
+	for _, item := range c.item {
+		if !item.IsDataDirty() {
+			items = append(items, item)
+		}
+	}
+
+	sort.Sort(items)
+
+	// Reset items until the quota is OK
+	for _, item := range items {
+		if c.used < quota {
+			break
+		}
+		resetResult, spaceFreed, err := item.Reset()
+		// The item space might be freed even if we get an error after the cache file is removed
+		// The item will not be removed or reset if the cache data is dirty (DataDirty)
+		c.used -= spaceFreed
+		fs.Infof(nil, "vfs cache purgeClean item.Reset %s: %s, freed %d bytes", item.GetName(), resetResult.String(), spaceFreed)
+		if resetResult == RemovedNotInUse {
+			delete(c.item, item.name)
+		}
+		if err != nil {
+			fs.Errorf(nil, "vfs cache purgeClean item.Reset %s reset failed, err = %v, freed %d bytes", item.GetName(), err, spaceFreed)
+			c.errItems[item.name] = err
 		}
 	}
+
+	// Resest outOfSpace without checking whether we have reduced cache space below the quota.
+	// This allows some files to reduce their pendingAccesses count to allow them to be reset
+	// in the next iteration of the purge cleaner loop.
+
+	c.outOfSpace = false
+	c.cond.Broadcast()
+}
+
+// purgeOld gets rid of any files that are over age
+func (c *Cache) purgeOld(maxAge time.Duration) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	// cutoff := time.Now().Add(-maxAge)
+	for _, item := range c.item {
+		c.removeNotInUse(item, maxAge, false)
+	}
+	if c.used < int64(c.opt.CacheMaxSize) {
+		c.outOfSpace = false
+		c.cond.Broadcast()
+	}
 }
 
 // Purge any empty directories
@@ -439,16 +549,8 @@ func (c *Cache) purgeEmptyDirs() {
 	}
 }
 
-// Remove any files that are over quota starting from the
-// oldest first
-func (c *Cache) purgeOverQuota(quota int64) {
-	c._purgeOverQuota(quota, func(item *Item) {
-		item.remove("over quota")
-	})
-}
-
 // updateUsed updates c.used so it is accurate
-func (c *Cache) updateUsed() {
+func (c *Cache) updateUsed() (used int64) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
@@ -457,15 +559,19 @@ func (c *Cache) updateUsed() {
 		newUsed += item.getDiskSize()
 	}
 	c.used = newUsed
+	return newUsed
 }
 
-func (c *Cache) _purgeOverQuota(quota int64, remove func(item *Item)) {
+// Remove clean cache files that are not open until the total space
+// is reduced below quota starting from the oldest first
+func (c *Cache) purgeOverQuota(quota int64) {
 	c.updateUsed()
 
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
 	if quota <= 0 || c.used < quota {
+
 		return
 	}
 
@@ -482,18 +588,16 @@ func (c *Cache) _purgeOverQuota(quota int64, remove func(item *Item)) {
 
 	// Remove items until the quota is OK
 	for _, item := range items {
-		if c.used < quota {
-			break
-		}
-		c.used -= item.getDiskSize()
-		remove(item)
-		// Remove the entry
-		delete(c.item, item.name)
+		c.removeNotInUse(item, 0, c.used <= quota)
+	}
+	if c.used < quota {
+		c.outOfSpace = false
+		c.cond.Broadcast()
 	}
 }
 
 // clean empties the cache of stuff if it can
-func (c *Cache) clean() {
+func (c *Cache) clean(removeCleanFiles bool) {
 	// Cache may be empty so end
 	_, err := os.Stat(c.root)
 	if os.IsNotExist(err) {
@@ -504,12 +608,37 @@ func (c *Cache) clean() {
 	oldItems, oldUsed := len(c.item), fs.SizeSuffix(c.used)
 	c.mu.Unlock()
 
-	// Remove any files that are over age
-	c.purgeOld(c.opt.CacheMaxAge)
+	// loop cleaning the cache until we reach below cache quota
+	for {
+		// Remove any files that are over age
+		c.purgeOld(c.opt.CacheMaxAge)
+
+		// Now remove files not in use until cache size is below quota starting from the
+		// oldest first
+		c.purgeOverQuota(int64(c.opt.CacheMaxSize))
+
+		// removeCleanFiles indicates that we got ENOSPC error
+		// We remove cache files that are not dirty if we are still avove the max cache size
+		if removeCleanFiles {
+			c.purgeClean(int64(c.opt.CacheMaxSize))
+			c.retryFailedResets()
+		} else {
+			break
+		}
 
-	// Now remove any files that are over quota starting from the
-	// oldest first
-	c.purgeOverQuota(int64(c.opt.CacheMaxSize))
+		used := c.updateUsed()
+		if used <= int64(c.opt.CacheMaxSize) && len(c.errItems) == 0 {
+			break
+		}
+	}
+
+	// Was kicked?
+	if removeCleanFiles {
+		c.kickerMu.Lock() // Make sure this is called with cache mutex unlocked
+		// Reenable io threads to kick me
+		c.cleanerKicked = false
+		c.kickerMu.Unlock()
+	}
 
 	// Stats
 	c.mu.Lock()
@@ -526,7 +655,7 @@ func (c *Cache) clean() {
 	fs.Infof(nil, "vfs cache: cleaned: objects %d (was %d) in use %d, to upload %d, uploading %d, total size %v (was %v)", newItems, oldItems, totalInUse, uploadsQueued, uploadsInProgress, newUsed, oldUsed)
 }
 
-// cleaner calls clean at regular intervals
+// cleaner calls clean at regular intervals and upon being kicked for out-of-space condition
 //
 // doesn't return until context is cancelled
 func (c *Cache) cleaner(ctx context.Context) {
@@ -535,14 +664,16 @@ func (c *Cache) cleaner(ctx context.Context) {
 		return
 	}
 	// Start cleaning the cache immediately
-	c.clean()
+	c.clean(false)
 	// Then every interval specified
 	timer := time.NewTicker(c.opt.CachePollInterval)
 	defer timer.Stop()
 	for {
 		select {
+		case <-c.kick: // a thread encountering ENOSPC kicked me
+			c.clean(true) // remove inUse files that are clean (!item.info.Dirty)
 		case <-timer.C:
-			c.clean()
+			c.clean(false) // do not remove inUse files
 		case <-ctx.Done():
 			fs.Debugf(nil, "vfs cache: cleaner exiting")
 			return
diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.go
index f2c22ee2c..b7cde1d9a 100644
--- a/vfs/vfscache/downloaders/downloaders.go
+++ b/vfs/vfscache/downloaders/downloaders.go
@@ -10,6 +10,7 @@ import (
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/asyncreader"
 	"github.com/rclone/rclone/fs/chunkedreader"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/lib/ranges"
 	"github.com/rclone/rclone/vfs/vfscommon"
 )
@@ -146,7 +147,9 @@ func (dls *Downloaders) _countErrors(n int64, err error) {
 		return
 	}
 	if err != nil {
+		//if err != syscall.ENOSPC {
 		dls.errorCount++
+		//}
 		dls.lastErr = err
 		fs.Infof(dls.src, "vfs cache: downloader: error count now %d: %v", dls.errorCount, err)
 	}
@@ -404,6 +407,11 @@ func (dls *Downloaders) kickWaiters() (err error) {
 			fs.Errorf(dls.src, "vfs cache: restart download failed: %v", err)
 		}
 	}
+	if fserrors.IsErrNoSpace(dls.lastErr) {
+		fs.Errorf(dls.src, "vfs cache: cache is out of space %d/%d: last error: %v", dls.errorCount, maxErrorCount, dls.lastErr)
+		dls._closeWaiters(dls.lastErr)
+		return dls.lastErr
+	}
 
 	if dls.errorCount > maxErrorCount {
 		fs.Errorf(dls.src, "vfs cache: too many errors %d/%d: last error: %v", dls.errorCount, maxErrorCount, dls.lastErr)
@@ -600,6 +608,7 @@ func (dl *downloader) download() (n int64, err error) {
 	if err != nil && errors.Cause(err) != asyncreader.ErrorStreamAbandoned {
 		return n, errors.Wrap(err, "vfs reader: failed to write to cache file")
 	}
+
 	return n, nil
 }
 
diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go
index 835151ef6..2f3749609 100644
--- a/vfs/vfscache/item.go
+++ b/vfs/vfscache/item.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/lib/ranges"
@@ -47,19 +48,20 @@ import (
 // The Info field is written to the backing store to store status
 type Item struct {
 	// read only
-	c *Cache // cache this is part of
-
-	mu          sync.Mutex               // protect the variables
-	name        string                   // name in the VFS
-	opens       int                      // number of times file is open
-	downloaders *downloaders.Downloaders // a record of the downloaders in action - may be nil
-	o           fs.Object                // object we are caching - may be nil
-	fd          *os.File                 // handle we are using to read and write to the file
-	metaDirty   bool                     // set if the info needs writeback
-	modified    bool                     // set if the file has been modified since the last Open
-	info        Info                     // info about the file to persist to backing store
-	writeBackID writeback.Handle         // id of any writebacks in progress
-
+	c               *Cache                   // cache this is part of
+	mu              sync.Mutex               // protect the variables
+	cond            *sync.Cond               // synchronize with cache cleaner
+	name            string                   // name in the VFS
+	opens           int                      // number of times file is open
+	downloaders     *downloaders.Downloaders // a record of the downloaders in action - may be nil
+	o               fs.Object                // object we are caching - may be nil
+	fd              *os.File                 // handle we are using to read and write to the file
+	metaDirty       bool                     // set if the info needs writeback
+	modified        bool                     // set if the file has been modified since the last Open
+	info            Info                     // info about the file to persist to backing store
+	writeBackID     writeback.Handle         // id of any writebacks in progress
+	pendingAccesses int                      // number of threads - cache reset not allowed if not zero
+	beingReset      bool                     // cache cleaner is resetting the cache file, access not allowed
 }
 
 // Info is persisted to backing store
@@ -75,6 +77,24 @@ type Info struct {
 // Items are a slice of *Item ordered by ATime
 type Items []*Item
 
+// ResetResult reports the actual action taken in the Reset function and reason
+type ResetResult int
+
+// Constants used to report actual action taken in the Reset function and reason
+const (
+	SkippedDirty         ResetResult = iota // Dirty item cannot be reset
+	SkippedPendingAccess                    // Reset pending access can lead to deadlock
+	SkippedEmpty                            // Reset empty item does not save space
+	RemovedNotInUse                         // Item not used. Remove instead of reset
+	ResetFailed                             // Reset failed with an error
+	ResetComplete                           // Reset completed successfully
+)
+
+func (rr ResetResult) String() string {
+	return [...]string{"Dirty item skipped", "In-access item skipped", "Empty item skipped",
+		"Not-in-use item removed", "Item reset failed", "Item reset completed"}[rr]
+}
+
 func (v Items) Len() int      { return len(v) }
 func (v Items) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
 func (v Items) Less(i, j int) bool {
@@ -112,7 +132,7 @@ func newItem(c *Cache, name string) (item *Item) {
 			ATime:   now,
 		},
 	}
-
+	item.cond = sync.NewCond(&item.mu)
 	// check the cache file exists
 	osPath := c.toOSPath(name)
 	fi, statErr := os.Stat(osPath)
@@ -340,6 +360,13 @@ func (item *Item) _getSize() (size int64, err error) {
 	return size, err
 }
 
+// GetName gets the vfs name of the item
+func (item *Item) GetName() (name string) {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+	return item.name
+}
+
 // GetSize gets the current size of the item
 func (item *Item) GetSize() (size int64, err error) {
 	item.mu.Lock()
@@ -399,15 +426,70 @@ func (item *Item) IsDirty() bool {
 	return item.metaDirty || item.info.Dirty
 }
 
+// IsDataDirty returns true if the item's data is dirty
+func (item *Item) IsDataDirty() bool {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+	return item.info.Dirty
+}
+
+// Create the cache file and store the metadata on disk
+// Called with item.mu locked
+func (item *Item) _createFile(osPath string) (err error) {
+	if item.fd != nil {
+		return errors.New("vfs cache item: internal error: didn't Close file")
+	}
+	item.modified = false
+	fd, err := file.OpenFile(osPath, os.O_RDWR, 0600)
+	if err != nil {
+		return errors.Wrap(err, "vfs cache item: open failed")
+	}
+	err = file.SetSparse(fd)
+	if err != nil {
+		fs.Debugf(item.name, "vfs cache: failed to set as a sparse file: %v", err)
+	}
+	item.fd = fd
+
+	err = item._save()
+	if err != nil {
+		closeErr := item.fd.Close()
+		if closeErr != nil {
+			fs.Errorf(item.name, "vfs cache: item.fd.Close: closeErr: %v", err)
+		}
+		item.fd = nil
+		return errors.Wrap(err, "vfs cache item: _save failed")
+	}
+	return err
+}
+
+// Open the local file from the object passed in.  Wraps open()
+// to provide recovery from out of space error.
+func (item *Item) Open(o fs.Object) (err error) {
+	for retries := 0; retries < fs.Config.LowLevelRetries; retries++ {
+		item.preAccess()
+		err = item.open(o)
+		item.postAccess()
+		if err == nil {
+			break
+		}
+		fs.Errorf(item.name, "vfs cache: failed to open item: %v", err)
+		if !fserrors.IsErrNoSpace(err) && err.Error() != "no space left on device" {
+			fs.Errorf(item.name, "Non-out-of-space error encountered during open")
+			break
+		}
+		item.c.KickCleaner()
+	}
+	return err
+}
+
 // Open the local file from the object passed in (which may be nil)
 // which implies we are about to create the file
-func (item *Item) Open(o fs.Object) (err error) {
+func (item *Item) open(o fs.Object) (err error) {
 	// defer log.Trace(o, "item=%p", item)("err=%v", &err)
 	item.mu.Lock()
 	defer item.mu.Unlock()
 
 	item.info.ATime = time.Now()
-	item.opens++
 
 	osPath, err := item.c.mkdir(item.name) // No locking in Cache
 	if err != nil {
@@ -419,29 +501,18 @@ func (item *Item) Open(o fs.Object) (err error) {
 		return errors.Wrap(err, "vfs cache item: check object failed")
 	}
 
+	item.opens++
 	if item.opens != 1 {
 		return nil
 	}
-	if item.fd != nil {
-		return errors.New("vfs cache item: internal error: didn't Close file")
-	}
-	item.modified = false
-
-	fd, err := file.OpenFile(osPath, os.O_RDWR, 0600)
-	if err != nil {
-		return errors.Wrap(err, "vfs cache item: open failed")
-	}
-	err = file.SetSparse(fd)
-	if err != nil {
-		fs.Debugf(item.name, "vfs cache: failed to set as a sparse file: %v", err)
-	}
-	item.fd = fd
 
-	err = item._save()
+	err = item._createFile(osPath)
 	if err != nil {
-		return err
+		item._remove("item.open failed on _createFile, remove cache data/metadata files")
+		item.fd = nil
+		item.opens--
+		return errors.Wrap(err, "vfs cache item: create cache file failed")
 	}
-
 	// Unlock the Item.mu so we can call some methods which take Cache.mu
 	item.mu.Unlock()
 
@@ -767,6 +838,197 @@ func (item *Item) remove(reason string) (wasWriting bool) {
 	return item._remove(reason)
 }
 
+// RemoveNotInUse is called to remove cache file that has not been accessed recently
+// It may also be called for removing empty cache files too when the quota is already reached.
+func (item *Item) RemoveNotInUse(maxAge time.Duration, emptyOnly bool) (removed bool, spaceFreed int64) {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+
+	spaceFreed = 0
+	removed = false
+
+	if item.opens != 0 || item.metaDirty || item.info.Dirty {
+		return
+	}
+
+	removeIt := false
+	if maxAge == 0 {
+		removeIt = true // quota-driven removal
+	}
+	if maxAge != 0 {
+		cutoff := time.Now().Add(-maxAge)
+		// If not locked and access time too long ago - delete the file
+		accessTime := item.info.ATime
+		if accessTime.Sub(cutoff) <= 0 {
+			removeIt = true
+		}
+	}
+	if removeIt {
+		spaceUsed := item.info.Rs.Size()
+		if !emptyOnly || spaceUsed == 0 {
+			spaceFreed = spaceUsed
+			removed = true
+			if item._remove("Removing old cache file not in use") {
+				fs.Errorf(item.name, "item removed when it was writing/uploaded")
+			}
+		}
+	}
+	return
+}
+
+// Reset is called by the cache purge functions only to reset (empty the contents) cache files that
+// are not dirty.  It is used when cache space runs out and we see some ENOSPC error.
+func (item *Item) Reset() (rr ResetResult, spaceFreed int64, err error) {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+
+	// The item is not being used now.  Just remove it instead of resetting it.
+	if item.opens == 0 && !item.metaDirty && !item.info.Dirty {
+		spaceFreed = item.info.Rs.Size()
+		if item._remove("Removing old cache file not in use") {
+			fs.Errorf(item.name, "item removed when it was writing/uploaded")
+		}
+		return RemovedNotInUse, spaceFreed, nil
+	}
+
+	// do not reset dirty file
+	if item.info.Dirty {
+		return SkippedDirty, 0, nil
+	}
+
+	/* A wait on pendingAccessCnt to become 0 can lead to deadlock when an item.Open bumps
+	   up the pendingAccesses count, calls item.open, which calls cache.put. The cache.put
+	   operation needs the cache mutex, which is held here.  We skip this file now. The
+	   caller (the cache cleaner thread) may retry resetting this item if the cache size does
+	   not reduce below quota. */
+	if item.pendingAccesses > 0 {
+		return SkippedPendingAccess, 0, nil
+	}
+
+	/* Do not need to reset an empty cache file unless it was being reset and the reset failed.
+	   Some thread(s) may be waiting on the reset's succesful completion in that case. */
+	if item.info.Rs.Size() == 0 && item.beingReset == false {
+		return SkippedEmpty, 0, nil
+	}
+
+	item.beingReset = true
+
+	/* Error handling from this point on (setting item.fd and item.beingReset):
+	   Since Reset is called by the cache cleaner thread, there is no direct way to return
+	   the error to the io threads.  Set item.fd to nil upon internal errors, so that the
+	   io threads will return internal errors seeing a nil fd. In the case when the error
+	   is ENOSPC, keep the item in isBeingReset state and that will keep the item.ReadAt
+	   waiting at its beginning. The cache purge loop will try to redo the reset after cache
+	   space is made available again. This recovery design should allow most io threads to
+	   eventually go through, unless large files are written/overwritten concurrently and
+	   the total size of these files exceed the cache storage limit. */
+
+	// Close the downloaders
+	// Accumulate and log errors
+	checkErr := func(e error) {
+		if e != nil {
+			fs.Errorf(item.o, "vfs cache: item reset failed: %v", e)
+			if err == nil {
+				err = e
+			}
+		}
+	}
+
+	if downloaders := item.downloaders; downloaders != nil {
+		item.downloaders = nil
+		// FIXME need to unlock to kill downloader - should we
+		// re-arrange locking so this isn't necessary?  maybe
+		// downloader should use the item mutex for locking? or put a
+		// finer lock on Rs?
+		//
+		// downloader.Write calls ensure which needs the lock
+		// close downloader with mutex unlocked
+		item.mu.Unlock()
+		checkErr(downloaders.Close(nil))
+		item.mu.Lock()
+	}
+
+	// close the file handle
+	// fd can be nil if we tried Reset and failed before because of ENOSPC during reset
+	if item.fd != nil {
+		checkErr(item.fd.Close())
+		if err != nil {
+			// Could not close the cache file
+			item.beingReset = false
+			item.cond.Broadcast()
+			return ResetFailed, 0, err
+		}
+		item.fd = nil
+	}
+
+	spaceFreed = item.info.Rs.Size()
+
+	// This should not be possible.  We get here only if cache data is not dirty.
+	if item._remove("cache out of space, item is clean") {
+		fs.Errorf(item.o, "vfs cache item removed when it was writing/uploaded")
+	}
+
+	// can we have an item with no dirty data (so that we can get here) and nil item.o at the same time?
+	fso := item.o
+	checkErr(item._checkObject(fso))
+	if err != nil {
+		item.beingReset = false
+		item.cond.Broadcast()
+		return ResetFailed, spaceFreed, err
+	}
+
+	osPath := item.c.toOSPath(item.name)
+	checkErr(item._createFile(osPath))
+	if err != nil {
+		item._remove("cache reset failed on _createFile, removed cache data file")
+		item.fd = nil // This allows a new Reset redo to have a clean state to deal with
+		if !fserrors.IsErrNoSpace(err) {
+			item.beingReset = false
+			item.cond.Broadcast()
+		}
+		return ResetFailed, spaceFreed, err
+	}
+
+	// Create the downloaders
+	if item.o != nil {
+		item.downloaders = downloaders.New(item, item.c.opt, item.name, item.o)
+	}
+
+	/* The item will stay in the beingReset state if we get an error that prevents us from
+	reaching this point.  The cache purge loop will redo the failed Reset. */
+	item.beingReset = false
+	item.cond.Broadcast()
+
+	return ResetComplete, spaceFreed, err
+}
+
+// ProtectCache either waits for an ongoing cache reset to finish or increases pendingReads
+// to protect against cache reset on this item while the thread potentially uses the cache file
+// Cache cleaner waits until pendingReads is zero before resetting cache.
+func (item *Item) preAccess() {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+
+	if item.beingReset {
+		for {
+			item.cond.Wait()
+			if !item.beingReset {
+				break
+			}
+		}
+	}
+	item.pendingAccesses++
+}
+
+// postAccess reduces the pendingReads count enabling cache reset upon ENOSPC
+func (item *Item) postAccess() {
+	item.mu.Lock()
+	defer item.mu.Unlock()
+
+	item.pendingAccesses--
+	item.cond.Broadcast()
+}
+
 // _present returns true if the whole file has been downloaded
 //
 // call with the lock held
@@ -811,6 +1073,10 @@ func (item *Item) _ensure(offset, size int64) (err error) {
 	}
 	r := ranges.Range{Pos: offset, Size: size}
 	present := item.info.Rs.Present(r)
+	/* This statement simulates a cache space error for test purpose */
+	/* if present != true && item.info.Rs.Size() > 32*1024*1024 {
+		return errors.New("no space left on device")
+	} */
 	fs.Debugf(nil, "vfs cache: looking for range=%+v in %+v - present %v", r, item.info.Rs, present)
 	item.mu.Unlock()
 	defer item.mu.Lock()
@@ -877,6 +1143,7 @@ func (item *Item) setModTime(modTime time.Time) {
 	item.mu.Lock()
 	item._updateFingerprint()
 	item._setModTime(modTime)
+	item.info.ModTime = modTime
 	err := item._save()
 	if err != nil {
 		fs.Errorf(item.name, "vfs cache: setModTime: failed to save item info: %v", err)
@@ -886,6 +1153,27 @@ func (item *Item) setModTime(modTime time.Time) {
 
 // ReadAt bytes from the file at off
 func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
+	n = 0
+	for retries := 0; retries < fs.Config.LowLevelRetries; retries++ {
+		item.preAccess()
+		n, err = item.readAt(b, off)
+		item.postAccess()
+		if err == nil {
+			break
+		}
+		fs.Errorf(item.name, "vfs cache: failed to _ensure cache %v", err)
+		if !fserrors.IsErrNoSpace(err) && err.Error() != "no space left on device" {
+			fs.Debugf(item.name, "vfs cache: failed to _ensure cache %v is not out of space", err)
+			break
+		}
+		item.c.KickCleaner()
+	}
+
+	return n, err
+}
+
+// ReadAt bytes from the file at off
+func (item *Item) readAt(b []byte, off int64) (n int, err error) {
 	item.mu.Lock()
 	if item.fd == nil {
 		item.mu.Unlock()
@@ -895,15 +1183,17 @@ func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
 		item.mu.Unlock()
 		return 0, io.EOF
 	}
+	defer item.mu.Unlock()
+
 	err = item._ensure(off, int64(len(b)))
 	if err != nil {
-		item.mu.Unlock()
-		return n, err
+		return 0, err
 	}
+
 	item.info.ATime = time.Now()
-	item.mu.Unlock()
-	// Do the reading with Item.mu unlocked
-	return item.fd.ReadAt(b, off)
+	// Do the reading with Item.mu unlocked and cache protected by preAccess
+	n, err = item.fd.ReadAt(b, off)
+	return n, err
 }
 
 // WriteAt bytes to the file at off


Test patch:
diff --git a/backend/union/union_test.go b/backend/union/union_test.go
index 98d34a68b..676088220 100644
--- a/backend/union/union_test.go
+++ b/backend/union/union_test.go
@@ -153,3 +153,29 @@ func TestPolicy2(t *testing.T) {
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
+
+func TestPolicy3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-policy31")
+	tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-policy32")
+	tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-policy33")
+	require.NoError(t, os.MkdirAll(tempdir1, 0744))
+	require.NoError(t, os.MkdirAll(tempdir2, 0744))
+	require.NoError(t, os.MkdirAll(tempdir3, 0744))
+	upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
+	name := "TestUnionPolicy3"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "union"},
+			{Name: name, Key: "upstreams", Value: upstreams},
+			{Name: name, Key: "action_policy", Value: "all"},
+			{Name: name, Key: "create_policy", Value: "all"},
+			{Name: name, Key: "search_policy", Value: "all"},
+		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}
diff --git a/bin/test-repeat-vfs.sh b/bin/test-repeat-vfs.sh
index 396b232c8..c9702a870 100755
--- a/bin/test-repeat-vfs.sh
+++ b/bin/test-repeat-vfs.sh
@@ -3,6 +3,9 @@
 
 set -e
 
+# Optionally set the iterations with the first parameter
+iterations=${1:-100}
+
 base=$(dirname $(dirname $(realpath "$0")))
 echo ${base}
 run=${base}/bin/test-repeat.sh
@@ -16,12 +19,8 @@ vfs/vfscache/downloaders
 cmd/cmount
 "
 
-testdirs="
-cmd/cmount
-"
-
 for testdir in ${testdirs}; do
-    echo "Testing ${testdir}"
+    echo "Testing ${testdir} with ${iterations} iterations"
     cd ${base}/${testdir}
-    ${run} -c=100 -race -tags=cmount
+    ${run} -i=${iterations} -race -tags=cmount
 done
diff --git a/fs/fspath/path_test.go b/fs/fspath/path_test.go
index 135708ec8..c07379030 100644
--- a/fs/fspath/path_test.go
+++ b/fs/fspath/path_test.go
@@ -2,6 +2,7 @@ package fspath
 
 import (
 	"fmt"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
@@ -153,6 +154,7 @@ func TestJoinRootPath(t *testing.T) {
 		{[]string{"", "//server/sub", "path"}, "//server/sub/path"},
 		{[]string{"", "//server", "//path"}, "//server/path"},
 		{[]string{"", "//server/sub", "//path"}, "//server/sub/path"},
+		{[]string{"", filepath.FromSlash("//server/sub"), filepath.FromSlash("//path")}, "//server/sub/path"},
 	} {
 		got := JoinRootPath(test.elements...)
 		assert.Equal(t, test.want, got)
diff --git a/fs/rc/internal_test.go b/fs/rc/internal_test.go
index 118215301..3eac2be57 100644
--- a/fs/rc/internal_test.go
+++ b/fs/rc/internal_test.go
@@ -2,6 +2,10 @@ package rc
 
 import (
 	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
 	"runtime"
 	"testing"
 
@@ -10,9 +14,17 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/version"
 )
 
+func TestMain(m *testing.M) {
+	// Pretend to be rclone version if we have a version string parameter
+	if os.Args[len(os.Args)-1] == "version" {
+		fmt.Printf("rclone %s\n", fs.Version)
+		os.Exit(0)
+	}
+	os.Exit(m.Run())
+}
+
 func TestInternalNoop(t *testing.T) {
 	call := Calls.Get("rc/noop")
 	assert.NotNil(t, call)
@@ -93,7 +105,7 @@ func TestCoreVersion(t *testing.T) {
 	assert.Equal(t, runtime.GOARCH, out["arch"])
 	assert.Equal(t, runtime.Version(), out["goVersion"])
 	_ = out["isGit"].(bool)
-	v := out["decomposed"].(version.Version)
+	v := out["decomposed"].([]int64)
 	assert.True(t, len(v) >= 2)
 }
 
@@ -119,3 +131,22 @@ func TestCoreQuit(t *testing.T) {
 	_, err := call.Fn(context.Background(), in)
 	require.Error(t, err)
 }
+
+// core/command: Runs a raw rclone command
+func TestCoreCommand(t *testing.T) {
+	call := Calls.Get("core/command")
+
+	var httpResponse http.ResponseWriter = httptest.NewRecorder()
+
+	in := Params{
+		"command":   "version",
+		"opt":       map[string]string{},
+		"arg":       []string{},
+		"_response": &httpResponse,
+	}
+	got, err := call.Fn(context.Background(), in)
+	require.NoError(t, err)
+
+	assert.Equal(t, fmt.Sprintf("rclone %s\n", fs.Version), got["result"])
+	assert.Equal(t, false, got["error"])
+}
diff --git a/fs/rc/webgui/rc_test.go b/fs/rc/webgui/rc_test.go
new file mode 100644
index 000000000..82b19ec10
--- /dev/null
+++ b/fs/rc/webgui/rc_test.go
@@ -0,0 +1,153 @@
+package webgui
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/rclone/rclone/fs/rc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testPluginName = "rclone-test-plugin"
+const testPluginAuthor = "rclone"
+const testPluginKey = testPluginAuthor + "/" + testPluginName
+const testPluginURL = "https://github.com/" + testPluginAuthor + "/" + testPluginName + "/"
+
+func setCacheDir(t *testing.T) string {
+	cacheDir, err := ioutil.TempDir("", "rclone-cache-dir")
+	assert.Nil(t, err)
+	PluginsPath = filepath.Join(cacheDir, "plugins")
+	pluginsConfigPath = filepath.Join(cacheDir, "config")
+
+	loadedPlugins = newPlugins(availablePluginsJSONPath)
+	err = loadedPlugins.readFromFile()
+	assert.Nil(t, err)
+	return cacheDir
+}
+
+func cleanCacheDir(t *testing.T, cacheDir string) {
+	_ = os.RemoveAll(cacheDir)
+}
+
+func addPlugin(t *testing.T) {
+	addPlugin := rc.Calls.Get("pluginsctl/addPlugin")
+	assert.NotNil(t, addPlugin)
+	in := rc.Params{
+		"url": testPluginURL,
+	}
+	out, err := addPlugin.Fn(context.Background(), in)
+	if err != nil && strings.Contains(err.Error(), "bad HTTP status") {
+		t.Skipf("skipping test as plugin download failed: %v", err)
+	}
+	require.Nil(t, err)
+	assert.Nil(t, out)
+
+}
+
+func removePlugin(t *testing.T) {
+	addPlugin := rc.Calls.Get("pluginsctl/removePlugin")
+	assert.NotNil(t, addPlugin)
+
+	in := rc.Params{
+		"name": testPluginKey,
+	}
+	out, err := addPlugin.Fn(context.Background(), in)
+	assert.NotNil(t, err)
+	assert.Nil(t, out)
+}
+
+//func TestListTestPlugins(t *testing.T) {
+//	addPlugin := rc.Calls.Get("pluginsctl/listTestPlugins")
+//	assert.NotNil(t, addPlugin)
+//	in := rc.Params{}
+//	out, err := addPlugin.Fn(context.Background(), in)
+//	assert.Nil(t, err)
+//	expected := rc.Params{
+//		"loadedTestPlugins": map[string]PackageJSON{},
+//	}
+//	assert.Equal(t, expected, out)
+//}
+
+//func TestRemoveTestPlugin(t *testing.T) {
+//	addPlugin := rc.Calls.Get("pluginsctl/removeTestPlugin")
+//	assert.NotNil(t, addPlugin)
+//	in := rc.Params{
+//		"name": "",
+//	}
+//	out, err := addPlugin.Fn(context.Background(), in)
+//	assert.NotNil(t, err)
+//	assert.Nil(t, out)
+//}
+
+func TestAddPlugin(t *testing.T) {
+	cacheDir := setCacheDir(t)
+	defer cleanCacheDir(t, cacheDir)
+
+	addPlugin(t)
+	_, ok := loadedPlugins.LoadedPlugins[testPluginKey]
+	assert.True(t, ok)
+
+	//removePlugin(t)
+	//_, ok = loadedPlugins.LoadedPlugins[testPluginKey]
+	//assert.False(t, ok)
+}
+
+func TestListPlugins(t *testing.T) {
+	cacheDir := setCacheDir(t)
+	defer cleanCacheDir(t, cacheDir)
+
+	addPlugin := rc.Calls.Get("pluginsctl/listPlugins")
+	assert.NotNil(t, addPlugin)
+	in := rc.Params{}
+	out, err := addPlugin.Fn(context.Background(), in)
+	assert.Nil(t, err)
+	expected := rc.Params{
+		"loadedPlugins":     map[string]PackageJSON{},
+		"loadedTestPlugins": map[string]PackageJSON{},
+	}
+	assert.Equal(t, expected, out)
+}
+
+func TestRemovePlugin(t *testing.T) {
+	cacheDir := setCacheDir(t)
+	defer cleanCacheDir(t, cacheDir)
+
+	addPlugin(t)
+	removePluginCall := rc.Calls.Get("pluginsctl/removePlugin")
+	assert.NotNil(t, removePlugin)
+
+	in := rc.Params{
+		"name": testPluginKey,
+	}
+	out, err := removePluginCall.Fn(context.Background(), in)
+	assert.Nil(t, err)
+	assert.Nil(t, out)
+	removePlugin(t)
+	assert.Equal(t, len(loadedPlugins.LoadedPlugins), 0)
+
+}
+
+func TestPluginsForType(t *testing.T) {
+	addPlugin := rc.Calls.Get("pluginsctl/getPluginsForType")
+	assert.NotNil(t, addPlugin)
+	in := rc.Params{
+		"type":       "",
+		"pluginType": "FileHandler",
+	}
+	out, err := addPlugin.Fn(context.Background(), in)
+	assert.Nil(t, err)
+	assert.NotNil(t, out)
+
+	in = rc.Params{
+		"type":       "video/mp4",
+		"pluginType": "",
+	}
+	_, err = addPlugin.Fn(context.Background(), in)
+	assert.Nil(t, err)
+	assert.NotNil(t, out)
+}
diff --git a/fs/version/version_test.go b/fs/version/version_test.go
deleted file mode 100644
index 42095b4aa..000000000
--- a/fs/version/version_test.go
+++ /dev/null
@@ -1,89 +0,0 @@
-package version
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNew(t *testing.T) {
-	for _, test := range []struct {
-		in      string
-		want    Version
-		wantErr bool
-	}{
-		{"v1.41", Version{1, 41}, false},
-		{"rclone v1.41", Version{1, 41}, false},
-		{"rclone v1.41.23", Version{1, 41, 23}, false},
-		{"rclone v1.41.23-100", Version{1, 41, 23, 100}, false},
-		{"rclone v1.41-100", Version{1, 41, 0, 100}, false},
-		{"rclone v1.41.23-100-g12312a", Version{1, 41, 23, 100}, false},
-		{"rclone v1.41-100-g12312a", Version{1, 41, 0, 100}, false},
-		{"rclone v1.42-005-g56e1e820β", Version{1, 42, 0, 5}, false},
-		{"rclone v1.42-005-g56e1e820-feature-branchβ", Version{1, 42, 0, 5}, false},
-
-		{"v1.41s", nil, true},
-		{"rclone v1-41", nil, true},
-		{"rclone v1.41.2c3", nil, true},
-		{"rclone v1.41.23-100 potato", nil, true},
-		{"rclone 1.41-100", nil, true},
-		{"rclone v1.41.23-100-12312a", nil, true},
-
-		{"v1.41-DEV", Version{1, 41, 999, 999}, false},
-	} {
-		what := fmt.Sprintf("in=%q", test.in)
-		got, err := New(test.in)
-		if test.wantErr {
-			assert.Error(t, err, what)
-		} else {
-			assert.NoError(t, err, what)
-		}
-		assert.Equal(t, test.want, got, what)
-	}
-
-}
-
-func TestCmp(t *testing.T) {
-	for _, test := range []struct {
-		a, b Version
-		want int
-	}{
-		{Version{1}, Version{1}, 0},
-		{Version{1}, Version{2}, -1},
-		{Version{2}, Version{1}, 1},
-		{Version{2}, Version{2, 1}, -1},
-		{Version{2, 1}, Version{2}, 1},
-		{Version{2, 1}, Version{2, 1}, 0},
-		{Version{2, 1}, Version{2, 2}, -1},
-		{Version{2, 2}, Version{2, 1}, 1},
-	} {
-		got := test.a.Cmp(test.b)
-		if got < 0 {
-			got = -1
-		} else if got > 0 {
-			got = 1
-		}
-		assert.Equal(t, test.want, got, fmt.Sprintf("%v cmp %v", test.a, test.b))
-		// test the reverse
-		got = -test.b.Cmp(test.a)
-		assert.Equal(t, test.want, got, fmt.Sprintf("%v cmp %v", test.b, test.a))
-	}
-}
-
-func TestString(t *testing.T) {
-	v, err := New("v1.44.1-2")
-	assert.NoError(t, err)
-
-	assert.Equal(t, "1.44.1.2", v.String())
-}
-
-func TestIsGit(t *testing.T) {
-	v, err := New("v1.44")
-	assert.NoError(t, err)
-	assert.Equal(t, false, v.IsGit())
-
-	v, err = New("v1.44-DEV")
-	assert.NoError(t, err)
-	assert.Equal(t, true, v.IsGit())
-}
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go
index 857686853..381d64ae1 100644
--- a/fstest/fstests/fstests.go
+++ b/fstest/fstests/fstests.go
@@ -335,7 +335,8 @@ func Run(t *testing.T, opt *Opt) {
 	// Return true if f (or any of the things it wraps) is bucket
 	// based but not at the root.
 	isBucketBasedButNotRoot := func(f fs.Fs) bool {
-		return fs.UnWrapFs(f).Features().BucketBased && strings.Contains(strings.Trim(f.Root(), "/"), "/")
+		f = fs.UnWrapFs(f)
+		return f.Features().BucketBased && strings.Contains(strings.Trim(f.Root(), "/"), "/")
 	}
 
 	// Initialise the remote
@@ -1874,6 +1875,9 @@ func Run(t *testing.T, opt *Opt) {
 		if !isBucketBasedButNotRoot(remote) {
 			err = operations.Purge(ctx, remote, "")
 			assert.Error(t, err, "Expecting error after on second purge")
+			if errors.Cause(err) != fs.ErrorDirNotFound {
+				t.Log("Warning: this should produce fs.ErrorDirNotFound")
+			}
 		}
 
 	})
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index d768e3970..8a2dd85d2 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -41,13 +41,15 @@ backends:
    remote:   "TestChunkerChunk3bNometaLocal:"
    fastlist: true
    maxfile:  6k
- - backend:  "chunker"
-   remote:   "TestChunkerMailru:"
-   fastlist: true
- - backend:  "chunker"
-   remote:   "TestChunkerChunk50bMailru:"
-   fastlist: true
-   maxfile:  10k
+ # Disable chunker with mailru tests until Mailru is fixed - see
+ # https://github.com/rclone/rclone/issues/4376
+ # - backend:  "chunker"
+ #   remote:   "TestChunkerMailru:"
+ #   fastlist: true
+ # - backend:  "chunker"
+ #   remote:   "TestChunkerChunk50bMailru:"
+ #   fastlist: true
+ #   maxfile:  10k
  - backend:  "chunker"
    remote:   "TestChunkerChunk50bYandex:"
    fastlist: true
@@ -133,12 +135,15 @@ backends:
      - TestIntegration/FsMkdir/FsEncoding/leading_HT
      - TestIntegration/FsMkdir/FsEncoding/leading_VT
      - TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0
- - backend:  "s3"
-   remote:   "TestS3DigitalOcean:"
-   fastlist: true
-   ignore:
-     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
-     - TestIntegration/FsMkdir/FsPutFiles/SetTier
+ # Disabled due to excessive rate limiting at DO which cause the tests never to pass
+ # This hits the rate limit as documented here: https://www.digitalocean.com/docs/spaces/#limits
+ # 2 COPYs per 5 minutes on any individual object in a Space
+ # - backend:  "s3"
+ #   remote:   "TestS3DigitalOcean:"
+ #   fastlist: true
+ #   ignore:
+ #     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
+ #     - TestIntegration/FsMkdir/FsPutFiles/SetTier
  - backend:  "s3"
    remote:   "TestS3Ceph:"
    fastlist: true
@@ -201,6 +206,8 @@ backends:
  - backend:  "fichier"
    remote:   "TestFichier:"
    fastlist: false
+   tests:
+     - backend
  - backend:  "qingstor"
    remote:   "TestQingStor:"
    fastlist: false
diff --git a/fstest/test_all/report.go b/fstest/test_all/report.go
index f1d4f5ec4..03ca3a518 100644
--- a/fstest/test_all/report.go
+++ b/fstest/test_all/report.go
@@ -140,7 +140,7 @@ func (r *Report) LogSummary() {
 	if !r.AllPassed() {
 		for _, t := range r.Failed {
 			log.Printf("  * %s", toShell(t.nextCmdLine()))
-			log.Printf("    * Failed tests: %v", t.failedTests)
+			log.Printf("    * Failed tests: %v", t.FailedTests)
 		}
 	}
 }
@@ -264,7 +264,7 @@ a:focus {
 <td>{{ if ne $prevRemote .Remote }}{{ .Remote }}{{ end }}{{ $prevRemote = .Remote }}</td>
 <td>{{ .Path }}</td>
 <td><span class="{{ .FastList }}">{{ .FastList }}</span></td>
-<td>{{ .FailedTests }}</td>
+<td>{{ .FailedTestsCSV }}</td>
 <td>{{ range $i, $v := .Logs }}<a href="{{ $v }}">#{{ $i }}</a> {{ end }}</td>
 </tr>
 {{ end }}
diff --git a/fstest/test_all/run.go b/fstest/test_all/run.go
index 083b5eb24..e4f8b2ee8 100644
--- a/fstest/test_all/run.go
+++ b/fstest/test_all/run.go
@@ -46,16 +46,16 @@ type Run struct {
 	SizeLimit int64  // maximum test file size
 	Ignore    map[string]struct{}
 	// Internals
-	cmdLine     []string
-	cmdString   string
-	try         int
+	CmdLine     []string
+	CmdString   string
+	Try         int
 	err         error
 	output      []byte
-	failedTests []string
-	runFlag     string
-	logDir      string   // directory to place the logs
-	trialName   string   // name/log file name of current trial
-	trialNames  []string // list of all the trials
+	FailedTests []string
+	RunFlag     string
+	LogDir      string   // directory to place the logs
+	TrialName   string   // name/log file name of current trial
+	TrialNames  []string // list of all the trials
 }
 
 // Runs records multiple Run objects
@@ -92,7 +92,7 @@ func (rs Runs) Less(i, j int) bool {
 // dumpOutput prints the error output
 func (r *Run) dumpOutput() {
 	log.Println("------------------------------------------------------------")
-	log.Printf("---- %q ----", r.cmdString)
+	log.Printf("---- %q ----", r.CmdString)
 	log.Println(string(r.output))
 	log.Println("------------------------------------------------------------")
 }
@@ -130,8 +130,8 @@ var failRe = regexp.MustCompile(`(?m)^\s*--- FAIL: (Test.*?) \(`)
 
 // findFailures looks for all the tests which failed
 func (r *Run) findFailures() {
-	oldFailedTests := r.failedTests
-	r.failedTests = nil
+	oldFailedTests := r.FailedTests
+	r.FailedTests = nil
 	excludeParents := map[string]struct{}{}
 	ignored := 0
 	for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
@@ -140,7 +140,7 @@ func (r *Run) findFailures() {
 		if _, found := r.Ignore[failedTest]; found {
 			ignored++
 		} else {
-			r.failedTests = append(r.failedTests, failedTest)
+			r.FailedTests = append(r.FailedTests, failedTest)
 		}
 		// Find all the parents of this test
 		parts := strings.Split(failedTest, "/")
@@ -149,50 +149,50 @@ func (r *Run) findFailures() {
 		}
 	}
 	// Exclude the parents
-	var newTests = r.failedTests[:0]
-	for _, failedTest := range r.failedTests {
+	var newTests = r.FailedTests[:0]
+	for _, failedTest := range r.FailedTests {
 		if _, excluded := excludeParents[failedTest]; !excluded {
 			newTests = append(newTests, failedTest)
 		}
 	}
-	r.failedTests = newTests
-	if len(r.failedTests) == 0 && ignored > 0 {
-		log.Printf("%q - Found %d ignored errors only - marking as good", r.cmdString, ignored)
+	r.FailedTests = newTests
+	if len(r.FailedTests) == 0 && ignored > 0 {
+		log.Printf("%q - Found %d ignored errors only - marking as good", r.CmdString, ignored)
 		r.err = nil
 		r.dumpOutput()
 		return
 	}
-	if len(r.failedTests) != 0 {
-		r.runFlag = testsToRegexp(r.failedTests)
+	if len(r.FailedTests) != 0 {
+		r.RunFlag = testsToRegexp(r.FailedTests)
 	} else {
-		r.runFlag = ""
+		r.RunFlag = ""
 	}
-	if r.passed() && len(r.failedTests) != 0 {
-		log.Printf("%q - Expecting no errors but got: %v", r.cmdString, r.failedTests)
+	if r.passed() && len(r.FailedTests) != 0 {
+		log.Printf("%q - Expecting no errors but got: %v", r.CmdString, r.FailedTests)
 		r.dumpOutput()
-	} else if !r.passed() && len(r.failedTests) == 0 {
-		log.Printf("%q - Expecting errors but got none: %v", r.cmdString, r.failedTests)
+	} else if !r.passed() && len(r.FailedTests) == 0 {
+		log.Printf("%q - Expecting errors but got none: %v", r.CmdString, r.FailedTests)
 		r.dumpOutput()
-		r.failedTests = oldFailedTests
+		r.FailedTests = oldFailedTests
 	}
 }
 
 // nextCmdLine returns the next command line
 func (r *Run) nextCmdLine() []string {
-	cmdLine := r.cmdLine
-	if r.runFlag != "" {
-		cmdLine = append(cmdLine, "-test.run", r.runFlag)
+	CmdLine := r.CmdLine
+	if r.RunFlag != "" {
+		CmdLine = append(CmdLine, "-test.run", r.RunFlag)
 	}
-	return cmdLine
+	return CmdLine
 }
 
 // trial runs a single test
 func (r *Run) trial() {
-	cmdLine := r.nextCmdLine()
-	cmdString := toShell(cmdLine)
-	msg := fmt.Sprintf("%q - Starting (try %d/%d)", cmdString, r.try, *maxTries)
+	CmdLine := r.nextCmdLine()
+	CmdString := toShell(CmdLine)
+	msg := fmt.Sprintf("%q - Starting (try %d/%d)", CmdString, r.Try, *maxTries)
 	log.Println(msg)
-	logName := path.Join(r.logDir, r.trialName)
+	logName := path.Join(r.LogDir, r.TrialName)
 	out, err := os.Create(logName)
 	if err != nil {
 		log.Fatalf("Couldn't create log file: %v", err)
@@ -207,7 +207,7 @@ func (r *Run) trial() {
 
 	// Early exit if --try-run
 	if *dryRun {
-		log.Printf("Not executing as --dry-run: %v", cmdLine)
+		log.Printf("Not executing as --dry-run: %v", CmdLine)
 		_, _ = fmt.Fprintln(out, "--dry-run is set - not running")
 		return
 	}
@@ -226,7 +226,7 @@ func (r *Run) trial() {
 	var b bytes.Buffer
 	multiOut := io.MultiWriter(out, &b)
 
-	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
+	cmd := exec.Command(CmdLine[0], CmdLine[1:]...)
 	cmd.Stderr = multiOut
 	cmd.Stdout = multiOut
 	cmd.Dir = r.Path
@@ -236,9 +236,9 @@ func (r *Run) trial() {
 	duration := time.Since(start)
 	r.findFailures()
 	if r.passed() {
-		msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, r.try, *maxTries)
+		msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", CmdString, duration, r.Try, *maxTries)
 	} else {
-		msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, r.try, *maxTries, r.err, r.failedTests)
+		msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", CmdString, duration, r.Try, *maxTries, r.err, r.FailedTests)
 	}
 	log.Println(msg)
 	_, _ = fmt.Fprintln(out, msg)
@@ -282,12 +282,12 @@ func (r *Run) MakeTestBinary() {
 	binary := r.BinaryPath()
 	binaryName := r.BinaryName()
 	log.Printf("%s: Making test binary %q", r.Path, binaryName)
-	cmdLine := []string{"go", "test", "-c"}
+	CmdLine := []string{"go", "test", "-c"}
 	if *dryRun {
-		log.Printf("Not executing: %v", cmdLine)
+		log.Printf("Not executing: %v", CmdLine)
 		return
 	}
-	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
+	cmd := exec.Command(CmdLine[0], CmdLine[1:]...)
 	cmd.Dir = r.Path
 	err := cmd.Run()
 	if err != nil {
@@ -320,7 +320,7 @@ func (r *Run) Name() string {
 	if r.FastList {
 		ns = append(ns, "fastlist")
 	}
-	ns = append(ns, fmt.Sprintf("%d", r.try))
+	ns = append(ns, fmt.Sprintf("%d", r.Try))
 	s := strings.Join(ns, "-")
 	s = strings.Replace(s, ":", "", -1)
 	return s
@@ -331,52 +331,52 @@ func (r *Run) Init() {
 	prefix := "-test."
 	if r.NoBinary {
 		prefix = "-"
-		r.cmdLine = []string{"go", "test"}
+		r.CmdLine = []string{"go", "test"}
 	} else {
-		r.cmdLine = []string{"./" + r.BinaryName()}
+		r.CmdLine = []string{"./" + r.BinaryName()}
 	}
-	r.cmdLine = append(r.cmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote)
+	r.CmdLine = append(r.CmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote)
 	if *listRetries > 0 {
-		r.cmdLine = append(r.cmdLine, "-list-retries", fmt.Sprint(*listRetries))
+		r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(*listRetries))
 	}
-	r.try = 1
+	r.Try = 1
 	if *verbose {
-		r.cmdLine = append(r.cmdLine, "-verbose")
+		r.CmdLine = append(r.CmdLine, "-verbose")
 		fs.Config.LogLevel = fs.LogLevelDebug
 	}
 	if *runOnly != "" {
-		r.cmdLine = append(r.cmdLine, prefix+"run", *runOnly)
+		r.CmdLine = append(r.CmdLine, prefix+"run", *runOnly)
 	}
 	if r.FastList {
-		r.cmdLine = append(r.cmdLine, "-fast-list")
+		r.CmdLine = append(r.CmdLine, "-fast-list")
 	}
 	if r.Short {
-		r.cmdLine = append(r.cmdLine, "-short")
+		r.CmdLine = append(r.CmdLine, "-short")
 	}
 	if r.SizeLimit > 0 {
-		r.cmdLine = append(r.cmdLine, "-size-limit", strconv.FormatInt(r.SizeLimit, 10))
+		r.CmdLine = append(r.CmdLine, "-size-limit", strconv.FormatInt(r.SizeLimit, 10))
 	}
-	r.cmdString = toShell(r.cmdLine)
+	r.CmdString = toShell(r.CmdLine)
 }
 
 // Logs returns all the log names
 func (r *Run) Logs() []string {
-	return r.trialNames
+	return r.TrialNames
 }
 
-// FailedTests returns the failed tests as a comma separated string, limiting the number
-func (r *Run) FailedTests() string {
+// FailedTestsCSV returns the failed tests as a comma separated string, limiting the number
+func (r *Run) FailedTestsCSV() string {
 	const maxTests = 5
-	ts := r.failedTests
+	ts := r.FailedTests
 	if len(ts) > maxTests {
 		ts = ts[:maxTests:maxTests]
-		ts = append(ts, fmt.Sprintf("… (%d more)", len(r.failedTests)-maxTests))
+		ts = append(ts, fmt.Sprintf("… (%d more)", len(r.FailedTests)-maxTests))
 	}
 	return strings.Join(ts, ", ")
 }
 
 // Run runs all the trials for this test
-func (r *Run) Run(logDir string, result chan<- *Run) {
+func (r *Run) Run(LogDir string, result chan<- *Run) {
 	if r.OneOnly {
 		oneOnlyMu.Lock()
 		mu := oneOnly[r.Backend]
@@ -389,11 +389,11 @@ func (r *Run) Run(logDir string, result chan<- *Run) {
 		defer mu.Unlock()
 	}
 	r.Init()
-	r.logDir = logDir
-	for r.try = 1; r.try <= *maxTries; r.try++ {
-		r.trialName = r.Name() + ".txt"
-		r.trialNames = append(r.trialNames, r.trialName)
-		log.Printf("Starting run with log %q", r.trialName)
+	r.LogDir = LogDir
+	for r.Try = 1; r.Try <= *maxTries; r.Try++ {
+		r.TrialName = r.Name() + ".txt"
+		r.TrialNames = append(r.TrialNames, r.TrialName)
+		log.Printf("Starting run with log %q", r.TrialName)
 		r.trial()
 		if r.passed() || r.NoRetries {
 			break
diff --git a/vfs/file_test.go b/vfs/file_test.go
index 59e30a778..0df50bfbd 100644
--- a/vfs/file_test.go
+++ b/vfs/file_test.go
@@ -265,9 +265,12 @@ func testFileRename(t *testing.T, mode vfscommon.CacheMode, inCache bool, forceC
 		require.NoError(t, err)
 		require.Equal(t, 14, n)
 
+		require.NoError(t, file.SetModTime(item.ModTime))
+
 		err = fd.Close()
 		require.NoError(t, err)
 	}
+	vfs.WaitForWriters(waitForWritersDelay)
 
 	// check file in cache
 	if inCache {
diff --git a/vfs/vfscache/cache_test.go b/vfs/vfscache/cache_test.go
index 4418b7993..e2b3f6d4b 100644
--- a/vfs/vfscache/cache_test.go
+++ b/vfs/vfscache/cache_test.go
@@ -33,6 +33,19 @@ func itemAsString(c *Cache) []string {
 	return out
 }
 
+// convert c.item to a string
+func itemSpaceAsString(c *Cache) []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var out []string
+	for name, item := range c.item {
+		space := item.info.Rs.Size()
+		out = append(out, fmt.Sprintf("name=%q opens=%d size=%d space=%d", filepath.ToSlash(name), item.opens, item.info.Size, space))
+	}
+	sort.Strings(out)
+	return out
+}
+
 // open an item and write to it
 func itemWrite(t *testing.T, item *Item, contents string) {
 	require.NoError(t, item.Open(nil))
@@ -174,7 +187,7 @@ func TestCacheNew(t *testing.T) {
 	assertPathNotExist(t, p)
 
 	// clean - have tested the internals already
-	c.clean()
+	c.clean(false)
 }
 
 func TestCacheOpens(t *testing.T) {
@@ -279,15 +292,7 @@ func TestCachePurgeOld(t *testing.T) {
 	defer cleanup()
 
 	// Test funcs
-	var removed []string
-	removeFile := func(item *Item) {
-		removed = append(removed, item.name)
-		item.remove("TestCachePurgeOld")
-	}
-
-	removed = nil
-	c._purgeOld(-10*time.Second, removeFile)
-	assert.Equal(t, []string(nil), removed)
+	c.purgeOld(-10 * time.Second)
 
 	potato2 := c.Item("sub/dir2/potato2")
 	require.NoError(t, potato2.Open(nil))
@@ -301,17 +306,23 @@ func TestCachePurgeOld(t *testing.T) {
 		`name="sub/dir2/potato2" opens=0 size=0`,
 	}, itemAsString(c))
 
-	removed = nil
-	c._purgeOld(-10*time.Second, removeFile)
+	c.purgeOld(-10 * time.Second)
+
 	assert.Equal(t, []string{
-		"sub/dir2/potato2",
-	}, removed)
+		`name="sub/dir/potato" opens=2 size=0`,
+	}, itemAsString(c))
 
 	require.NoError(t, potato.Close(nil))
 
-	removed = nil
-	c._purgeOld(-10*time.Second, removeFile)
-	assert.Equal(t, []string(nil), removed)
+	assert.Equal(t, []string{
+		`name="sub/dir/potato" opens=1 size=0`,
+	}, itemAsString(c))
+
+	c.purgeOld(-10 * time.Second)
+
+	assert.Equal(t, []string{
+		`name="sub/dir/potato" opens=1 size=0`,
+	}, itemAsString(c))
 
 	require.NoError(t, potato.Close(nil))
 
@@ -319,19 +330,13 @@ func TestCachePurgeOld(t *testing.T) {
 		`name="sub/dir/potato" opens=0 size=0`,
 	}, itemAsString(c))
 
-	removed = nil
-	c._purgeOld(10*time.Second, removeFile)
-	assert.Equal(t, []string(nil), removed)
+	c.purgeOld(10 * time.Second)
 
 	assert.Equal(t, []string{
 		`name="sub/dir/potato" opens=0 size=0`,
 	}, itemAsString(c))
 
-	removed = nil
-	c._purgeOld(-10*time.Second, removeFile)
-	assert.Equal(t, []string{
-		"sub/dir/potato",
-	}, removed)
+	c.purgeOld(-10 * time.Second)
 
 	assert.Equal(t, []string(nil), itemAsString(c))
 }
@@ -341,23 +346,6 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	defer cleanup()
 
 	// Test funcs
-	var removed []string
-	remove := func(item *Item) {
-		removed = append(removed, item.name)
-		item.remove("TestCachePurgeOverQuota")
-	}
-
-	removed = nil
-	c._purgeOverQuota(-1, remove)
-	assert.Equal(t, []string(nil), removed)
-
-	removed = nil
-	c._purgeOverQuota(0, remove)
-	assert.Equal(t, []string(nil), removed)
-
-	removed = nil
-	c._purgeOverQuota(1, remove)
-	assert.Equal(t, []string(nil), removed)
 
 	// Make some test files
 	potato := c.Item("sub/dir/potato")
@@ -372,9 +360,7 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	}, itemAsString(c))
 
 	// Check nothing removed
-	removed = nil
-	c._purgeOverQuota(1, remove)
-	assert.Equal(t, []string(nil), removed)
+	c.purgeOverQuota(1)
 
 	// Close the files
 	require.NoError(t, potato.Close(nil))
@@ -393,11 +379,7 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	potato2.info.ATime = t1
 
 	// Check only potato removed to get below quota
-	removed = nil
-	c._purgeOverQuota(10, remove)
-	assert.Equal(t, []string{
-		"sub/dir/potato",
-	}, removed)
+	c.purgeOverQuota(10)
 	assert.Equal(t, int64(6), c.used)
 
 	assert.Equal(t, []string{
@@ -423,11 +405,7 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	potato.info.ATime = t2
 
 	// Check only potato2 removed to get below quota
-	removed = nil
-	c._purgeOverQuota(10, remove)
-	assert.Equal(t, []string{
-		"sub/dir2/potato2",
-	}, removed)
+	c.purgeOverQuota(10)
 	assert.Equal(t, int64(5), c.used)
 	c.purgeEmptyDirs()
 
@@ -436,22 +414,82 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	}, itemAsString(c))
 
 	// Now purge everything
-	removed = nil
-	c._purgeOverQuota(1, remove)
-	assert.Equal(t, []string{
-		"sub/dir/potato",
-	}, removed)
+	c.purgeOverQuota(1)
 	assert.Equal(t, int64(0), c.used)
 	c.purgeEmptyDirs()
 
 	assert.Equal(t, []string(nil), itemAsString(c))
 
 	// Check nothing left behind
-	c.clean()
+	c.clean(false)
 	assert.Equal(t, int64(0), c.used)
 	assert.Equal(t, []string(nil), itemAsString(c))
 }
 
+// test reset clean files
+func TestCachePurgeClean(t *testing.T) {
+	r, c, cleanup := newItemTestCache(t)
+	defer cleanup()
+	contents, obj, patato1 := newFile(t, r, c, "existing")
+	_ = contents
+
+	// Open the object to create metadata for it
+	require.NoError(t, patato1.Open(obj))
+	require.NoError(t, patato1.Open(obj))
+
+	size, err := patato1.GetSize()
+	require.NoError(t, err)
+	assert.Equal(t, int64(100), size)
+
+	// Read something to instantiate the cache file
+	buf := make([]byte, 10)
+	_, err = patato1.ReadAt(buf, 10)
+	require.NoError(t, err)
+
+	// Test cache file present
+	_, err = os.Stat(patato1.c.toOSPath(patato1.name))
+	require.NoError(t, err)
+
+	// Add some potatos
+	potato2 := c.Item("sub/dir/potato2")
+	require.NoError(t, potato2.Open(nil))
+	require.NoError(t, potato2.Truncate(5))
+
+	potato3 := c.Item("sub/dir/potato3")
+	require.NoError(t, potato3.Open(nil))
+	require.NoError(t, potato3.Truncate(6))
+
+	c.updateUsed()
+	c.purgeClean(1)
+	assert.Equal(t, []string{
+		`name="existing" opens=2 size=100 space=0`,
+		`name="sub/dir/potato2" opens=1 size=5 space=5`,
+		`name="sub/dir/potato3" opens=1 size=6 space=6`,
+	}, itemSpaceAsString(c))
+	assert.Equal(t, int64(11), c.used)
+
+	require.NoError(t, potato2.Close(nil))
+	c.purgeClean(1)
+	assert.Equal(t, []string{
+		`name="existing" opens=2 size=100 space=0`,
+		`name="sub/dir/potato3" opens=1 size=6 space=6`,
+	}, itemSpaceAsString(c))
+	assert.Equal(t, int64(6), c.used)
+
+	require.NoError(t, patato1.Close(nil))
+	require.NoError(t, patato1.Close(nil))
+	require.NoError(t, potato3.Close(nil))
+
+	// Remove all files now.  The are all not in use.
+	// purgeClean does not remove empty cache files. purgeOverQuota does.
+	// So we use purgeOverQuota here for the cleanup.
+	c.purgeOverQuota(1)
+
+	c.purgeEmptyDirs()
+
+	assert.Equal(t, []string(nil), itemAsString(c))
+}
+
 func TestCacheInUse(t *testing.T) {
 	_, c, cleanup := newTestCache(t)
 	defer cleanup()