Repository: ncw/rclone
Pull request #3288 (2019-06-24 03:31:28)

Implement --compare-dest & --copy-dest
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?
<!--
Describe the changes here
-->
Implement rsync's --compare-dest & --copy-dest flags to complement the existing --backup-dir to help with incremental backup strategies.

#### Was the change discussed in an issue or in the forum before?

<!--
Link issues and relevant forum posts here.
-->
Discussed in #3278 

#### Checklist

This depends on my other PR #3272 to work with --backup-dir

- [X] I have read the [contribution guidelines](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [X] I have added tests for all changes in this PR if appropriate.
- [X] I have added documentation for the changes if appropriate.
- [X] All commit messages are in [house style](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [X] I'm done, this Pull Request is ready for review :-)


yparitcher: sorry for the sloppy code, i am not so used to the Go syntax
ncw: > sorry for the sloppy code, i am not so used to the Go syntax

No apologies necessary - I think you are doing very well and I really appreciate your submissions :-)
yparitcher: refactored, wrote better tests & found a few corner cases.
please review.
ncw: Just kicked the CI (yet another flakey test!) will review in a bit
yparitcher: you are right, I didn't realize some remotes only support one, will fix.
yparitcher: Now the test doesn't run on the `local` remote as it does not support copy only move
ncw: > Now the test doesn't run on the `local` remote as it does not support copy only move

Hmm, that is a little unfortunate but not the end of the world...

You'll need to add a test for Features().Copy and Skip the test if it isn't found to get the tests to pass.

I'm going to be away on vacation for a while, so communication will be sporadic !
yparitcher: should be ready

Patch:
diff --git a/Makefile b/Makefile
index 833de26b6..871d94f8d 100644
--- a/Makefile
+++ b/Makefile
@@ -151,7 +151,7 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
+	go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(TAG)
 
 appveyor_upload:
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -172,7 +172,7 @@ ifeq ($(TRAVIS_OS_NAME),linux)
 	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
 endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
-	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)
+	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG)
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
 ifndef BRANCH_PATH
 	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
diff --git a/backend/fichier/api.go b/backend/fichier/api.go
index 0f46b5b5c..b7514b192 100644
--- a/backend/fichier/api.go
+++ b/backend/fichier/api.go
@@ -88,7 +88,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
 	return entries, nil
 }
 
-func (f *Fs) listFiles(directoryID string) (filesList *FilesList, err error) {
+func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) {
 	// fs.Debugf(f, "Requesting files for dir `%s`", directoryID)
 	request := ListFilesRequest{
 		FolderID: directoryID,
@@ -111,7 +111,7 @@ func (f *Fs) listFiles(directoryID string) (filesList *FilesList, err error) {
 	return filesList, nil
 }
 
-func (f *Fs) listFolders(directoryID string) (foldersList *FoldersList, err error) {
+func (f *Fs) listFolders(directoryID int) (foldersList *FoldersList, err error) {
 	// fs.Debugf(f, "Requesting folders for id `%s`", directoryID)
 
 	request := ListFolderRequest{
@@ -148,12 +148,17 @@ func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, er
 		return nil, err
 	}
 
-	files, err := f.listFiles(directoryID)
+	folderID, err := strconv.Atoi(directoryID)
 	if err != nil {
 		return nil, err
 	}
 
-	folders, err := f.listFolders(directoryID)
+	files, err := f.listFiles(folderID)
+	if err != nil {
+		return nil, err
+	}
+
+	folders, err := f.listFolders(folderID)
 	if err != nil {
 		return nil, err
 	}
@@ -199,12 +204,12 @@ func getRemote(dir, fileName string) string {
 	return dir + "/" + fileName
 }
 
-func (f *Fs) makeFolder(leaf, directoryID string) (response *MakeFolderResponse, err error) {
+func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse, err error) {
 	name := replaceReservedChars(leaf)
 	// fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID)
 
 	request := MakeFolderRequest{
-		FolderID: directoryID,
+		FolderID: folderID,
 		Name:     name,
 	}
 
@@ -227,11 +232,11 @@ func (f *Fs) makeFolder(leaf, directoryID string) (response *MakeFolderResponse,
 	return response, err
 }
 
-func (f *Fs) removeFolder(name, directoryID string) (response *GenericOKResponse, err error) {
+func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKResponse, err error) {
 	// fs.Debugf(f, "Removing folder with id `%s`", directoryID)
 
 	request := &RemoveFolderRequest{
-		FolderID: directoryID,
+		FolderID: folderID,
 	}
 
 	opts := rest.Opts{
diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go
index b474bcea1..0a18b3210 100644
--- a/backend/fichier/fichier.go
+++ b/backend/fichier/fichier.go
@@ -70,7 +70,11 @@ type Fs struct {
 
 // FindLeaf finds a directory of name leaf in the folder with ID pathID
 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
-	folders, err := f.listFolders(pathID)
+	folderID, err := strconv.Atoi(pathID)
+	if err != nil {
+		return "", false, err
+	}
+	folders, err := f.listFolders(folderID)
 	if err != nil {
 		return "", false, err
 	}
@@ -87,7 +91,11 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 
 // CreateDir makes a directory with pathID as parent and name leaf
 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
-	resp, err := f.makeFolder(leaf, pathID)
+	folderID, err := strconv.Atoi(pathID)
+	if err != nil {
+		return "", err
+	}
+	resp, err := f.makeFolder(leaf, folderID)
 	if err != nil {
 		return "", err
 	}
@@ -239,7 +247,11 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 		return nil, err
 	}
 
-	files, err := f.listFiles(directoryID)
+	folderID, err := strconv.Atoi(directoryID)
+	if err != nil {
+		return nil, err
+	}
+	files, err := f.listFiles(folderID)
 	if err != nil {
 		return nil, err
 	}
@@ -371,12 +383,17 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		return err
 	}
 
-	did, err := f.dirCache.FindDir(ctx, dir, false)
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return err
+	}
+
+	folderID, err := strconv.Atoi(directoryID)
 	if err != nil {
 		return err
 	}
 
-	_, err = f.removeFolder(dir, did)
+	_, err = f.removeFolder(dir, folderID)
 	if err != nil {
 		return err
 	}
diff --git a/backend/fichier/structs.go b/backend/fichier/structs.go
index ef08b906c..b5487b59f 100644
--- a/backend/fichier/structs.go
+++ b/backend/fichier/structs.go
@@ -2,12 +2,12 @@ package fichier
 
 // ListFolderRequest is the request structure of the corresponding request
 type ListFolderRequest struct {
-	FolderID string `json:"folder_id"`
+	FolderID int `json:"folder_id"`
 }
 
 // ListFilesRequest is the request structure of the corresponding request
 type ListFilesRequest struct {
-	FolderID string `json:"folder_id"`
+	FolderID int `json:"folder_id"`
 }
 
 // DownloadRequest is the request structure of the corresponding request
@@ -18,7 +18,7 @@ type DownloadRequest struct {
 
 // RemoveFolderRequest is the request structure of the corresponding request
 type RemoveFolderRequest struct {
-	FolderID string `json:"folder_id"`
+	FolderID int `json:"folder_id"`
 }
 
 // RemoveFileRequest is the request structure of the corresponding request
@@ -40,7 +40,7 @@ type GenericOKResponse struct {
 // MakeFolderRequest is the request structure of the corresponding request
 type MakeFolderRequest struct {
 	Name     string `json:"name"`
-	FolderID string `json:"folder_id"`
+	FolderID int    `json:"folder_id"`
 }
 
 // MakeFolderResponse is the response structure of the corresponding request
@@ -108,12 +108,12 @@ type Folder struct {
 	CreateDate string `json:"create_date"`
 	ID         int    `json:"id"`
 	Name       string `json:"name"`
-	Pass       string `json:"pass"`
+	Pass       int    `json:"pass"`
 }
 
 // FoldersList is the structure how 1Fichier returns a list of Folders
 type FoldersList struct {
-	FolderID   string   `json:"folder_id"`
+	FolderID   int      `json:"folder_id"`
 	Name       string   `json:"name"`
 	Status     string   `json:"Status"`
 	SubFolders []Folder `json:"sub_folders"`
diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index d7b921f26..25457b130 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -86,7 +86,7 @@ when the ssh-agent contains many keys.`,
 			Default: false,
 		}, {
 			Name:    "use_insecure_cipher",
-			Help:    "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
+			Help:    "Enable the use of the aes128-cbc cipher and diffie-hellman-group-exchange-sha256, diffie-hellman-group-exchange-sha1 key exchange. Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.",
 			Default: false,
 			Examples: []fs.OptionExample{
 				{
@@ -94,7 +94,7 @@ when the ssh-agent contains many keys.`,
 					Help:  "Use default Cipher list.",
 				}, {
 					Value: "true",
-					Help:  "Enables the use of the aes128-cbc cipher.",
+					Help:  "Enables the use of the aes128-cbc cipher and diffie-hellman-group-exchange-sha256, diffie-hellman-group-exchange-sha1 key exchange.",
 				},
 			},
 		}, {
@@ -345,6 +345,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if opt.UseInsecureCipher {
 		sshConfig.Config.SetDefaults()
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
+		sshConfig.Config.KeyExchanges = append(sshConfig.Config.KeyExchanges, "diffie-hellman-group-exchange-sha1", "diffie-hellman-group-exchange-sha256")
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)
diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go
index 6517d06fe..b20151143 100644
--- a/backend/webdav/webdav.go
+++ b/backend/webdav/webdav.go
@@ -15,6 +15,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os/exec"
 	"path"
 	"strings"
 	"time"
@@ -80,17 +81,22 @@ func init() {
 		}, {
 			Name: "bearer_token",
 			Help: "Bearer token instead of user/pass (eg a Macaroon)",
+		}, {
+			Name:     "bearer_token_command",
+			Help:     "Command to run to get a bearer token",
+			Advanced: true,
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	URL         string `config:"url"`
-	Vendor      string `config:"vendor"`
-	User        string `config:"user"`
-	Pass        string `config:"pass"`
-	BearerToken string `config:"bearer_token"`
+	URL                string `config:"url"`
+	Vendor             string `config:"vendor"`
+	User               string `config:"user"`
+	Pass               string `config:"pass"`
+	BearerToken        string `config:"bearer_token"`
+	BearerTokenCommand string `config:"bearer_token_command"`
 }
 
 // Fs represents a remote webdav
@@ -158,7 +164,16 @@ var retryErrorCodes = []int{
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+	// If we have a bearer token command and it has expired then refresh it
+	if f.opt.BearerTokenCommand != "" && resp != nil && resp.StatusCode == 401 {
+		fs.Debugf(f, "Bearer token expired: %v", err)
+		authErr := f.fetchAndSetBearerToken()
+		if authErr != nil {
+			err = authErr
+		}
+		return true, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
@@ -207,7 +222,7 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(&opts, nil, &result)
-		return shouldRetry(resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// does not exist
@@ -330,7 +345,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if opt.User != "" || opt.Pass != "" {
 		f.srv.SetUserPass(opt.User, opt.Pass)
 	} else if opt.BearerToken != "" {
-		f.srv.SetHeader("Authorization", "BEARER "+opt.BearerToken)
+		f.setBearerToken(opt.BearerToken)
+	} else if f.opt.BearerTokenCommand != "" {
+		err = f.fetchAndSetBearerToken()
+		if err != nil {
+			return nil, err
+		}
 	}
 	f.srv.SetErrorHandler(errorHandler)
 	err = f.setQuirks(opt.Vendor)
@@ -360,6 +380,49 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return f, nil
 }
 
+// sets the BearerToken up
+func (f *Fs) setBearerToken(token string) {
+	f.opt.BearerToken = token
+	f.srv.SetHeader("Authorization", "BEARER "+token)
+}
+
+// fetch the bearer token using the command
+func (f *Fs) fetchBearerToken(cmd string) (string, error) {
+	var (
+		args   = strings.Split(cmd, " ")
+		stdout bytes.Buffer
+		stderr bytes.Buffer
+		c      = exec.Command(args[0], args[1:]...)
+	)
+	c.Stdout = &stdout
+	c.Stderr = &stderr
+	var (
+		err          = c.Run()
+		stdoutString = strings.TrimSpace(stdout.String())
+		stderrString = strings.TrimSpace(stderr.String())
+	)
+	if err != nil {
+		if stderrString == "" {
+			stderrString = stdoutString
+		}
+		return "", errors.Wrapf(err, "failed to get bearer token using %q: %s", f.opt.BearerTokenCommand, stderrString)
+	}
+	return stdoutString, nil
+}
+
+// fetch the bearer token and set it if successful
+func (f *Fs) fetchAndSetBearerToken() error {
+	if f.opt.BearerTokenCommand == "" {
+		return nil
+	}
+	token, err := f.fetchBearerToken(f.opt.BearerTokenCommand)
+	if err != nil {
+		return err
+	}
+	f.setBearerToken(token)
+	return nil
+}
+
 // setQuirks adjusts the Fs for the vendor passed in
 func (f *Fs) setQuirks(vendor string) error {
 	switch vendor {
@@ -480,7 +543,7 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(&opts, nil, &result)
-		return shouldRetry(resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
@@ -649,7 +712,7 @@ func (f *Fs) _mkdir(dirPath string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(&opts)
-		return shouldRetry(resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// already exists
@@ -713,7 +776,7 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(&opts, nil, nil)
-		return shouldRetry(resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "rmdir failed")
@@ -776,7 +839,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(&opts)
-		return shouldRetry(resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "Copy call failed")
@@ -872,7 +935,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(&opts)
-		return shouldRetry(resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "DirMove MOVE call failed")
@@ -913,7 +976,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(&opts, nil, &q)
-		return shouldRetry(resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "about call failed")
@@ -1033,7 +1096,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.Call(&opts)
-		return shouldRetry(resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, err
@@ -1081,7 +1144,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		resp, err = o.fs.srv.Call(&opts)
-		return shouldRetry(resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
 		// Give the WebDAV server a chance to get its internal state in order after the
@@ -1108,7 +1171,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	}
 	return o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.Call(&opts)
-		return shouldRetry(resp, err)
+		return o.fs.shouldRetry(resp, err)
 	})
 }
 
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 278c2b894..5d14b1647 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -328,6 +328,8 @@ If running rclone from a script you might want to use today's date as
 the directory name passed to `--backup-dir` to store the old files, or
 you might want to pass `--suffix` with today's date.
 
+See `--compare-dest` and `--copy-dest`.
+
 ### --bind string ###
 
 Local address to bind to for outgoing connections.  This can be an
@@ -447,6 +449,18 @@ quicker than without the `--checksum` flag.
 When using this flag, rclone won't update mtimes of remote files if
 they are incorrect as it would normally.
 
+### --compare-dest=DIR ###
+
+When using `sync`, `copy` or `move` DIR is checked in addition to the 
+destination for files. If a file identical to the source is found that 
+file is NOT copied from source. This is useful to copy just files that 
+have changed since the last backup.
+
+You must use the same remote as the destination of the sync.  The 
+compare directory must not overlap the destination directory.
+
+See `--copy-dest` and `--backup-dir`.
+
 ### --config=CONFIG_FILE ###
 
 Specify the location of the rclone config file.
@@ -475,6 +489,19 @@ The connection timeout is the amount of time rclone will wait for a
 connection to go through to a remote object storage system.  It is
 `1m` by default.
 
+### --copy-dest=DIR ###
+
+When using `sync`, `copy` or `move` DIR is checked in addition to the 
+destination for files. If a file identical to the source is found that 
+file is server side copied from DIR to the destination. This is useful 
+for incremental backup.
+
+The remote in use must support server side copy and you must
+use the same remote as the destination of the sync.  The compare
+directory must not overlap the destination directory.
+
+See `--compare-dest` and `--backup-dir`.
+
 ### --dedupe-mode MODE ###
 
 Mode to run dedupe command in.  One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`.  The default is `interactive`.  See the dedupe command for more information as to what these options mean.
diff --git a/docs/content/sftp.md b/docs/content/sftp.md
index 05fc177d7..a54a1bd63 100644
--- a/docs/content/sftp.md
+++ b/docs/content/sftp.md
@@ -75,22 +75,22 @@ host> example.com
 SSH username, leave blank for current username, ncw
 user> sftpuser
 SSH port, leave blank to use default (22)
-port> 
+port>
 SSH password, leave blank to use ssh-agent.
 y) Yes type in my own password
 g) Generate random password
 n) No leave this optional password blank
 y/g/n> n
 Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.
-key_file> 
+key_file>
 Remote config
 --------------------
 [remote]
 host = example.com
 user = sftpuser
-port = 
-pass = 
-key_file = 
+port =
+pass =
+key_file =
 --------------------
 y) Yes this is OK
 e) Edit this remote
@@ -243,7 +243,7 @@ when the ssh-agent contains many keys.
 
 #### --sftp-use-insecure-cipher
 
-Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.
+Enable the use of the aes128-cbc cipher and diffie-hellman-group-exchange-sha256, diffie-hellman-group-exchange-sha1 key exchange. Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.
 
 - Config:      use_insecure_cipher
 - Env Var:     RCLONE_SFTP_USE_INSECURE_CIPHER
@@ -253,7 +253,7 @@ Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow p
     - "false"
         - Use default Cipher list.
     - "true"
-        - Enables the use of the aes128-cbc cipher.
+        - Enables the use of the aes128-cbc cipher and diffie-hellman-group-exchange-sha256, diffie-hellman-group-exchange-sha1 key exchange.
 
 #### --sftp-disable-hashcheck
 
@@ -325,7 +325,7 @@ return the total space, free space, and used space on the remote
 for the disk of the specified path on the remote or, if not set,
 the disk of the root on the remote.
 `about` will fail if it does not have shell
-access or if `df` is not in the remote's PATH. 
+access or if `df` is not in the remote's PATH.
 
 Note that some SFTP servers (eg Synology) the paths are different for
 SSH and SFTP so the hashes can't be calculated properly.  For them
diff --git a/docs/content/webdav.md b/docs/content/webdav.md
index e8a7f277b..d9e389273 100644
--- a/docs/content/webdav.md
+++ b/docs/content/webdav.md
@@ -63,7 +63,7 @@ password:
 Confirm the password:
 password:
 Bearer token instead of user/pass (eg a Macaroon)
-bearer_token> 
+bearer_token>
 Remote config
 --------------------
 [remote]
@@ -72,7 +72,7 @@ url = https://example.com/remote.php/webdav/
 vendor = nextcloud
 user = user
 pass = *** ENCRYPTED ***
-bearer_token = 
+bearer_token =
 --------------------
 y) Yes this is OK
 e) Edit this remote
@@ -257,8 +257,13 @@ pass = encryptedpassword
 
 ### dCache ###
 
-[dCache](https://www.dcache.org/) is a storage system with WebDAV doors that support, beside basic and x509, 
-authentication with [Macaroons](https://www.dcache.org/manuals/workshop-2017-05-29-Umea/000-Final/anupam_macaroons_v02.pdf) (bearer tokens).
+dCache is a storage system that supports many protocols and
+authentication/authorisation schemes.  For WebDAV clients, it allows
+users to authenticate with username and password (BASIC), X.509,
+Kerberos, and various bearer tokens, including
+[Macaroons](https://www.dcache.org/manuals/workshop-2017-05-29-Umea/000-Final/anupam_macaroons_v02.pdf)
+and [OpenID-Connect](https://en.wikipedia.org/wiki/OpenID_Connect)
+access tokens.
 
 Configure as normal using the `other` type.  Don't enter a username or
 password, instead enter your Macaroon as the `bearer_token`.
@@ -277,3 +282,52 @@ bearer_token = your-macaroon
 
 There is a [script](https://github.com/sara-nl/GridScripts/blob/master/get-macaroon) that
 obtains a Macaroon from a dCache WebDAV endpoint, and creates an rclone config file.
+
+Macaroons may also be obtained from the dCacheView
+web-browser/JavaScript client that comes with dCache.
+
+### OpenID-Connect ###
+
+dCache also supports authenticating with OpenID-Connect access tokens.
+OpenID-Connect is a protocol (based on OAuth 2.0) that allows services
+to identify users who have authenticated with some central service.
+
+Support for OpenID-Connect in rclone is currently achieved using
+another software package called
+[oidc-agent](https://github.com/indigo-dc/oidc-agent).  This is a
+command-line tool that facilitates obtaining an access token.  Once
+installed and configured, an access token is obtained by running the
+`oidc-token` command.  The following example shows a (shortened)
+access token obtained from the *XDC* OIDC Provider.
+
+```
+paul@celebrimbor:~$ oidc-token XDC
+eyJraWQ[...]QFXDt0
+paul@celebrimbor:~$
+```
+
+**Note** Before the `oidc-token` command will work, the refresh token
+must be loaded into the oidc agent.  This is done with the `oidc-add`
+command (e.g., `oidc-add XDC`).  This is typically done once per login
+session.  Full details on this and how to register oidc-agent with
+your OIDC Provider are provided in the [oidc-agent
+documentation](https://indigo-dc.gitbooks.io/oidc-agent/).
+
+The rclone `bearer_token_command` configuration option is used to
+fetch the access token from oidc-agent.
+
+Configure as a normal WebDAV endpoint, using the 'other' vendor,
+leaving the username and password empty.  When prompted, choose to
+edit the advanced config and enter the command to get a bearer token
+(e.g., `oidc-agent XDC`).
+
+The following example config shows a WebDAV endpoint that uses
+oidc-agent to supply an access token from the *XDC* OIDC Provider.
+
+```
+[dcache]
+type = webdav
+url = https://dcache.example.org/
+vendor = other
+bearer_token_command = oidc-token XDC
+```
diff --git a/fs/config.go b/fs/config.go
index 6b77fc700..6817df55a 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -66,6 +66,8 @@ type ConfigInfo struct {
 	NoTraverse             bool
 	NoUpdateModTime        bool
 	DataRateUnit           string
+	CompareDest            string
+	CopyDest               string
 	BackupDir              string
 	Suffix                 string
 	SuffixKeepExtension    bool
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index 97e5cbe11..99c9fead7 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -67,6 +67,8 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing")
 	flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
 	flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
+	flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "use DIR to server side copy flies from.")
+	flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Compare dest to DIR also.")
 	flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.")
 	flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix to add to changed files.")
 	flags.BoolVarP(flagSet, &fs.Config.SuffixKeepExtension, "suffix-keep-extension", "", fs.Config.SuffixKeepExtension, "Preserve the extension when using --suffix.")
@@ -152,6 +154,10 @@ func SetFlags() {
 		log.Fatalf(`Can't use --size-only and --ignore-size together.`)
 	}
 
+	if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" {
+		log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
+	}
+
 	switch {
 	case len(fs.Config.StatsOneLineDateFormat) > 0:
 		fs.Config.StatsOneLineDate = true
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index b629c715c..3529a847a 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -96,7 +96,7 @@ func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal b
 // Otherwise the file is considered to be not equal including if there
 // were errors reading info.
 func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
-	return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
+	return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum, !fs.Config.NoUpdateModTime)
 }
 
 // sizeDiffers compare the size of src and dst taking into account the
@@ -110,7 +110,7 @@ func sizeDiffers(src, dst fs.ObjectInfo) bool {
 
 var checksumWarning sync.Once
 
-func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
+func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum, UpdateModTime bool) bool {
 	if sizeDiffers(src, dst) {
 		fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
 		return false
@@ -169,7 +169,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, chec
 	}
 
 	// mod time differs but hash is the same to reset mod time if required
-	if !fs.Config.NoUpdateModTime {
+	if UpdateModTime {
 		if fs.Config.DryRun {
 			fs.Logf(src, "Not updating modification time as --dry-run")
 		} else {
@@ -1360,6 +1360,115 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
 	return nil
 }
 
+// GetCompareDest sets up --compare-dest
+func GetCompareDest() (CompareDest fs.Fs, err error) {
+	CompareDest, err = cache.Get(fs.Config.CompareDest)
+	if err != nil {
+		return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", fs.Config.CompareDest, err))
+	}
+	return CompareDest, nil
+}
+
+// compareDest checks --compare-dest to see if src needs to
+// be copied
+//
+// Returns True if src is in --compare-dest
+func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (NoNeedTransfer bool, err error) {
+	var remote string
+	if dst == nil {
+		remote = src.Remote()
+	} else {
+		remote = dst.Remote()
+	}
+	CompareDestFile, err := CompareDest.NewObject(ctx, remote)
+	switch err {
+	case fs.ErrorObjectNotFound:
+		return false, nil
+	case nil:
+		break
+	default:
+		return false, err
+	}
+	if Equal(ctx, src, CompareDestFile) {
+		fs.Debugf(src, "Destination found in --compare-dest, skipping")
+		return true, nil
+	}
+	return false, nil
+}
+
+// GetCopyDest sets up --copy-dest
+func GetCopyDest(fdst fs.Fs) (CopyDest fs.Fs, err error) {
+	CopyDest, err = cache.Get(fs.Config.CopyDest)
+	if err != nil {
+		return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", fs.Config.CopyDest, err))
+	}
+	if !SameConfig(fdst, CopyDest) {
+		return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
+	}
+	if CopyDest.Features().Copy == nil {
+		return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy"))
+	}
+	return CopyDest, nil
+}
+
+// copyDest checks --copy-dest to see if src needs to
+// be copied
+//
+// Returns True if src was copied from --copy-dest
+func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
+	var remote string
+	if dst == nil {
+		remote = src.Remote()
+	} else {
+		remote = dst.Remote()
+	}
+	CopyDestFile, err := CopyDest.NewObject(ctx, remote)
+	switch err {
+	case fs.ErrorObjectNotFound:
+		return false, nil
+	case nil:
+		break
+	default:
+		return false, err
+	}
+	if equal(ctx, src, CopyDestFile, fs.Config.SizeOnly, fs.Config.CheckSum, false) {
+		if dst == nil || !Equal(ctx, src, dst) {
+			if dst != nil && backupDir != nil {
+				err = MoveBackupDir(ctx, backupDir, dst)
+				if err != nil {
+					return false, errors.Wrap(err, "moving to --backup-dir failed")
+				}
+				// If successful zero out the dstObj as it is no longer there
+				dst = nil
+			}
+			_, err := Copy(ctx, fdst, dst, remote, CopyDestFile)
+			if err != nil {
+				fs.Errorf(src, "Destination found in --copy-dest, error copying")
+				return false, nil
+			}
+			fs.Debugf(src, "Destination found in --copy-dest, using server side copy")
+			return true, nil
+		}
+		fs.Debugf(src, "Unchanged skipping")
+		return true, nil
+	}
+	fs.Debugf(src, "Destination not found in --copy-dest")
+	return false, nil
+}
+
+// CompareOrCopyDest checks --compare-dest and --copy-dest to see if src
+// does not need to be copied
+//
+// Returns True if src does not need to be copied
+func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
+	if fs.Config.CompareDest != "" {
+		return compareDest(ctx, dst, src, CompareOrCopyDest)
+	} else if fs.Config.CopyDest != "" {
+		return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir)
+	}
+	return false, nil
+}
+
 // NeedTransfer checks to see if src needs to be copied to dst using
 // the current config.
 //
@@ -1577,13 +1686,31 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
 		return err
 	}
 
-	if NeedTransfer(ctx, dstObj, srcObj) {
+	var backupDir, copyDestDir fs.Fs
+	if fs.Config.BackupDir != "" || fs.Config.Suffix != "" {
+		backupDir, err = BackupDir(fdst, fsrc, srcFileName)
+		if err != nil {
+			return errors.Wrap(err, "creating Fs for --backup-dir failed")
+		}
+	}
+	if fs.Config.CompareDest != "" {
+		copyDestDir, err = GetCompareDest()
+		if err != nil {
+			return err
+		}
+	} else if fs.Config.CopyDest != "" {
+		copyDestDir, err = GetCopyDest(fdst)
+		if err != nil {
+			return err
+		}
+	}
+	NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir)
+	if err != nil {
+		return err
+	}
+	if !NoNeedTransfer && NeedTransfer(ctx, dstObj, srcObj) {
 		// If destination already exists, then we must move it into --backup-dir if required
-		if dstObj != nil && (fs.Config.BackupDir != "" || fs.Config.Suffix != "") {
-			backupDir, err := BackupDir(fdst, fsrc, srcFileName)
-			if err != nil {
-				return errors.Wrap(err, "creating Fs for --backup-dir failed")
-			}
+		if dstObj != nil && backupDir != nil {
 			err = MoveBackupDir(ctx, backupDir, dstObj)
 			if err != nil {
 				return errors.Wrap(err, "moving to --backup-dir failed")
diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go
index c51a1fc75..1a33be81a 100644
--- a/fs/rc/rcserver/rcserver.go
+++ b/fs/rc/rcserver/rcserver.go
@@ -182,6 +182,8 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
 		return
 	}
 
+	delete(in, "_async") // remove the async parameter after parsing so vfs operations don't get confused
+
 	fs.Debugf(nil, "rc: %q: with parameters %+v", path, in)
 	var out rc.Params
 	if isAsync {
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 903998847..f96bdcd5b 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -28,39 +28,40 @@ type syncCopyMove struct {
 	deleteEmptySrcDirs bool
 	dir                string
 	// internal state
-	ctx            context.Context        // internal context for controlling go-routines
-	cancel         func()                 // cancel the context
-	noTraverse     bool                   // if set don't traverse the dst
-	deletersWg     sync.WaitGroup         // for delete before go routine
-	deleteFilesCh  chan fs.Object         // channel to receive deletes if delete before
-	trackRenames   bool                   // set if we should do server side renames
-	dstFilesMu     sync.Mutex             // protect dstFiles
-	dstFiles       map[string]fs.Object   // dst files, always filled
-	srcFiles       map[string]fs.Object   // src files, only used if deleteBefore
-	srcFilesChan   chan fs.Object         // passes src objects
-	srcFilesResult chan error             // error result of src listing
-	dstFilesResult chan error             // error result of dst listing
-	dstEmptyDirsMu sync.Mutex             // protect dstEmptyDirs
-	dstEmptyDirs   map[string]fs.DirEntry // potentially empty directories
-	srcEmptyDirsMu sync.Mutex             // protect srcEmptyDirs
-	srcEmptyDirs   map[string]fs.DirEntry // potentially empty directories
-	checkerWg      sync.WaitGroup         // wait for checkers
-	toBeChecked    *pipe                  // checkers channel
-	transfersWg    sync.WaitGroup         // wait for transfers
-	toBeUploaded   *pipe                  // copiers channel
-	errorMu        sync.Mutex             // Mutex covering the errors variables
-	err            error                  // normal error from copy process
-	noRetryErr     error                  // error with NoRetry set
-	fatalErr       error                  // fatal error
-	commonHash     hash.Type              // common hash type between src and dst
-	renameMapMu    sync.Mutex             // mutex to protect the below
-	renameMap      map[string][]fs.Object // dst files by hash - only used by trackRenames
-	renamerWg      sync.WaitGroup         // wait for renamers
-	toBeRenamed    *pipe                  // renamers channel
-	trackRenamesWg sync.WaitGroup         // wg for background track renames
-	trackRenamesCh chan fs.Object         // objects are pumped in here
-	renameCheck    []fs.Object            // accumulate files to check for rename here
-	backupDir      fs.Fs                  // place to store overwrites/deletes
+	ctx             context.Context        // internal context for controlling go-routines
+	cancel          func()                 // cancel the context
+	noTraverse      bool                   // if set don't traverse the dst
+	deletersWg      sync.WaitGroup         // for delete before go routine
+	deleteFilesCh   chan fs.Object         // channel to receive deletes if delete before
+	trackRenames    bool                   // set if we should do server side renames
+	dstFilesMu      sync.Mutex             // protect dstFiles
+	dstFiles        map[string]fs.Object   // dst files, always filled
+	srcFiles        map[string]fs.Object   // src files, only used if deleteBefore
+	srcFilesChan    chan fs.Object         // passes src objects
+	srcFilesResult  chan error             // error result of src listing
+	dstFilesResult  chan error             // error result of dst listing
+	dstEmptyDirsMu  sync.Mutex             // protect dstEmptyDirs
+	dstEmptyDirs    map[string]fs.DirEntry // potentially empty directories
+	srcEmptyDirsMu  sync.Mutex             // protect srcEmptyDirs
+	srcEmptyDirs    map[string]fs.DirEntry // potentially empty directories
+	checkerWg       sync.WaitGroup         // wait for checkers
+	toBeChecked     *pipe                  // checkers channel
+	transfersWg     sync.WaitGroup         // wait for transfers
+	toBeUploaded    *pipe                  // copiers channel
+	errorMu         sync.Mutex             // Mutex covering the errors variables
+	err             error                  // normal error from copy process
+	noRetryErr      error                  // error with NoRetry set
+	fatalErr        error                  // fatal error
+	commonHash      hash.Type              // common hash type between src and dst
+	renameMapMu     sync.Mutex             // mutex to protect the below
+	renameMap       map[string][]fs.Object // dst files by hash - only used by trackRenames
+	renamerWg       sync.WaitGroup         // wait for renamers
+	toBeRenamed     *pipe                  // renamers channel
+	trackRenamesWg  sync.WaitGroup         // wg for background track renames
+	trackRenamesCh  chan fs.Object         // objects are pumped in here
+	renameCheck     []fs.Object            // accumulate files to check for rename here
+	compareCopyDest fs.Fs                  // place to check for files to server side copy
+	backupDir       fs.Fs                  // place to store overwrites/deletes
 }
 
 func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) {
@@ -127,6 +128,19 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 			return nil, err
 		}
 	}
+	if fs.Config.CompareDest != "" {
+		var err error
+		s.compareCopyDest, err = operations.GetCompareDest()
+		if err != nil {
+			return nil, err
+		}
+	} else if fs.Config.CopyDest != "" {
+		var err error
+		s.compareCopyDest, err = operations.GetCopyDest(fdst)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return s, nil
 }
 
@@ -204,7 +218,11 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) {
 		accounting.Stats.Checking(src.Remote())
 		// Check to see if can store this
 		if src.Storable() {
-			if operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) {
+			NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir)
+			if err != nil {
+				s.processError(err)
+			}
+			if !NoNeedTransfer && operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) {
 				// If files are treated as immutable, fail if destination exists and does not match
 				if fs.Config.Immutable && pair.Dst != nil {
 					fs.Errorf(pair.Dst, "Source and destination exist but do not match: immutable file modified")
@@ -764,10 +782,17 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
 			case s.trackRenamesCh <- x:
 			}
 		} else {
-			// No need to check since doesn't exist
-			ok := s.toBeUploaded.Put(s.ctx, fs.ObjectPair{Src: x, Dst: nil})
-			if !ok {
-				return
+			// Check CompareDest && CopyDest
+			NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, nil, x, s.compareCopyDest, s.backupDir)
+			if err != nil {
+				s.processError(err)
+			}
+			if !NoNeedTransfer {
+				// No need to check since doesn't exist
+				ok := s.toBeUploaded.Put(s.ctx, fs.ObjectPair{Src: x, Dst: nil})
+				if !ok {
+					return
+				}
 			}
 		}
 	case fs.Directory:


Test patch:
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 44dd9cbf9..75f84373a 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -866,6 +866,183 @@ func TestCopyFileBackupDir(t *testing.T) {
 	fstest.CheckItems(t, r.Fremote, file1old, file1)
 }
 
+// Test with CompareDest set
+func TestCopyFileCompareDest(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	fs.Config.CompareDest = r.FremoteName + "/CompareDest"
+	defer func() {
+		fs.Config.CompareDest = ""
+	}()
+	fdst, err := fs.NewFs(r.FremoteName + "/dst")
+	require.NoError(t, err)
+
+	// check empty dest, empty compare
+	file1 := r.WriteFile("one", "one", t1)
+	fstest.CheckItems(t, r.Flocal, file1)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1.Path, file1.Path)
+	require.NoError(t, err)
+
+	file1dst := file1
+	file1dst.Path = "dst/one"
+
+	fstest.CheckItems(t, r.Fremote, file1dst)
+
+	// check old dest, empty compare
+	file1b := r.WriteFile("one", "onet2", t2)
+	fstest.CheckItems(t, r.Fremote, file1dst)
+	fstest.CheckItems(t, r.Flocal, file1b)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1b.Path, file1b.Path)
+	require.NoError(t, err)
+
+	file1bdst := file1b
+	file1bdst.Path = "dst/one"
+
+	fstest.CheckItems(t, r.Fremote, file1bdst)
+
+	// check old dest, new compare
+	file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
+	file2 := r.WriteObject(context.Background(), "CompareDest/one", "onet2", t2)
+	file1c := r.WriteFile("one", "onet2", t2)
+	fstest.CheckItems(t, r.Fremote, file2, file3)
+	fstest.CheckItems(t, r.Flocal, file1c)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1c.Path, file1c.Path)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Fremote, file2, file3)
+
+	// check empty dest, new compare
+	file4 := r.WriteObject(context.Background(), "CompareDest/two", "two", t2)
+	file5 := r.WriteFile("two", "two", t2)
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4)
+	fstest.CheckItems(t, r.Flocal, file1c, file5)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4)
+
+	// check new dest, new compare
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4)
+
+	// check empty dest, old compare
+	file5b := r.WriteFile("two", "twot3", t3)
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4)
+	fstest.CheckItems(t, r.Flocal, file1c, file5b)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5b.Path, file5b.Path)
+	require.NoError(t, err)
+
+	file5bdst := file5b
+	file5bdst.Path = "dst/two"
+
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
+}
+
+// Test with CopyDest set
+func TestCopyFileCopyDest(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	if r.Fremote.Features().Copy == nil {
+		t.Skip("Skipping test as remote does not support server side copy")
+	}
+
+	fs.Config.CopyDest = r.FremoteName + "/CopyDest"
+	defer func() {
+		fs.Config.CopyDest = ""
+	}()
+
+	fdst, err := fs.NewFs(r.FremoteName + "/dst")
+	require.NoError(t, err)
+
+	// check empty dest, empty copy
+	file1 := r.WriteFile("one", "one", t1)
+	fstest.CheckItems(t, r.Flocal, file1)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1.Path, file1.Path)
+	require.NoError(t, err)
+
+	file1dst := file1
+	file1dst.Path = "dst/one"
+
+	fstest.CheckItems(t, r.Fremote, file1dst)
+
+	// check old dest, empty copy
+	file1b := r.WriteFile("one", "onet2", t2)
+	fstest.CheckItems(t, r.Fremote, file1dst)
+	fstest.CheckItems(t, r.Flocal, file1b)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1b.Path, file1b.Path)
+	require.NoError(t, err)
+
+	file1bdst := file1b
+	file1bdst.Path = "dst/one"
+
+	fstest.CheckItems(t, r.Fremote, file1bdst)
+
+	// check old dest, new copy, backup-dir
+
+	fs.Config.BackupDir = r.FremoteName + "/BackupDir"
+
+	file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
+	file2 := r.WriteObject(context.Background(), "CopyDest/one", "onet2", t2)
+	file1c := r.WriteFile("one", "onet2", t2)
+	fstest.CheckItems(t, r.Fremote, file2, file3)
+	fstest.CheckItems(t, r.Flocal, file1c)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1c.Path, file1c.Path)
+	require.NoError(t, err)
+
+	file2dst := file2
+	file2dst.Path = "dst/one"
+	file3.Path = "BackupDir/one"
+
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3)
+	fs.Config.BackupDir = ""
+
+	// check empty dest, new copy
+	file4 := r.WriteObject(context.Background(), "CopyDest/two", "two", t2)
+	file5 := r.WriteFile("two", "two", t2)
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4)
+	fstest.CheckItems(t, r.Flocal, file1c, file5)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
+	require.NoError(t, err)
+
+	file4dst := file4
+	file4dst.Path = "dst/two"
+
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
+
+	// check new dest, new copy
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
+
+	// check empty dest, old copy
+	file6 := r.WriteObject(context.Background(), "CopyDest/three", "three", t2)
+	file7 := r.WriteFile("three", "threet3", t3)
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6)
+	fstest.CheckItems(t, r.Flocal, file1c, file5, file7)
+
+	err = operations.CopyFile(context.Background(), fdst, r.Flocal, file7.Path, file7.Path)
+	require.NoError(t, err)
+
+	file7dst := file7
+	file7dst.Path = "dst/three"
+
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst)
+}
+
 // testFsInfo is for unit testing fs.Info
 type testFsInfo struct {
 	name      string
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 6c9929b81..d721601ed 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -1216,6 +1216,196 @@ func TestSyncOverlap(t *testing.T) {
 	checkErr(Sync(context.Background(), FremoteSync, FremoteSync, false))
 }
 
+// Test with CompareDest set
+func TestSyncCompareDest(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	fs.Config.CompareDest = r.FremoteName + "/CompareDest"
+	defer func() {
+		fs.Config.CompareDest = ""
+	}()
+
+	fdst, err := fs.NewFs(r.FremoteName + "/dst")
+	require.NoError(t, err)
+
+	// check empty dest, empty compare
+	file1 := r.WriteFile("one", "one", t1)
+	fstest.CheckItems(t, r.Flocal, file1)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	file1dst := file1
+	file1dst.Path = "dst/one"
+
+	fstest.CheckItems(t, r.Fremote, file1dst)
+
+	// check old dest, empty compare
+	file1b := r.WriteFile("one", "onet2", t2)
+	fstest.CheckItems(t, r.Fremote, file1dst)
+	fstest.CheckItems(t, r.Flocal, file1b)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	file1bdst := file1b
+	file1bdst.Path = "dst/one"
+
+	fstest.CheckItems(t, r.Fremote, file1bdst)
+
+	// check old dest, new compare
+	file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
+	file2 := r.WriteObject(context.Background(), "CompareDest/one", "onet2", t2)
+	file1c := r.WriteFile("one", "onet2", t2)
+	fstest.CheckItems(t, r.Fremote, file2, file3)
+	fstest.CheckItems(t, r.Flocal, file1c)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Fremote, file2, file3)
+
+	// check empty dest, new compare
+	file4 := r.WriteObject(context.Background(), "CompareDest/two", "two", t2)
+	file5 := r.WriteFile("two", "two", t2)
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4)
+	fstest.CheckItems(t, r.Flocal, file1c, file5)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4)
+
+	// check new dest, new compare
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4)
+
+	// check empty dest, old compare
+	file5b := r.WriteFile("two", "twot3", t3)
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4)
+	fstest.CheckItems(t, r.Flocal, file1c, file5b)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	file5bdst := file5b
+	file5bdst.Path = "dst/two"
+
+	fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
+}
+
+// Test with CopyDest set
+func TestSyncCopyDest(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	if r.Fremote.Features().Copy == nil {
+		t.Skip("Skipping test as remote does not support server side copy")
+	}
+
+	fs.Config.CopyDest = r.FremoteName + "/CopyDest"
+	defer func() {
+		fs.Config.CopyDest = ""
+	}()
+
+	fdst, err := fs.NewFs(r.FremoteName + "/dst")
+	require.NoError(t, err)
+
+	// check empty dest, empty copy
+	file1 := r.WriteFile("one", "one", t1)
+	fstest.CheckItems(t, r.Flocal, file1)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	file1dst := file1
+	file1dst.Path = "dst/one"
+
+	fstest.CheckItems(t, r.Fremote, file1dst)
+
+	// check old dest, empty copy
+	file1b := r.WriteFile("one", "onet2", t2)
+	fstest.CheckItems(t, r.Fremote, file1dst)
+	fstest.CheckItems(t, r.Flocal, file1b)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	file1bdst := file1b
+	file1bdst.Path = "dst/one"
+
+	fstest.CheckItems(t, r.Fremote, file1bdst)
+
+	// check old dest, new copy, backup-dir
+
+	fs.Config.BackupDir = r.FremoteName + "/BackupDir"
+
+	file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
+	file2 := r.WriteObject(context.Background(), "CopyDest/one", "onet2", t2)
+	file1c := r.WriteFile("one", "onet2", t2)
+	fstest.CheckItems(t, r.Fremote, file2, file3)
+	fstest.CheckItems(t, r.Flocal, file1c)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	file2dst := file2
+	file2dst.Path = "dst/one"
+	file3.Path = "BackupDir/one"
+
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3)
+	fs.Config.BackupDir = ""
+
+	// check empty dest, new copy
+	file4 := r.WriteObject(context.Background(), "CopyDest/two", "two", t2)
+	file5 := r.WriteFile("two", "two", t2)
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4)
+	fstest.CheckItems(t, r.Flocal, file1c, file5)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	file4dst := file4
+	file4dst.Path = "dst/two"
+
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
+
+	// check new dest, new copy
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
+
+	// check empty dest, old copy
+	file6 := r.WriteObject(context.Background(), "CopyDest/three", "three", t2)
+	file7 := r.WriteFile("three", "threet3", t3)
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6)
+	fstest.CheckItems(t, r.Flocal, file1c, file5, file7)
+
+	accounting.Stats.ResetCounters()
+	err = Sync(context.Background(), fdst, r.Flocal, false)
+	require.NoError(t, err)
+
+	file7dst := file7
+	file7dst.Path = "dst/three"
+
+	fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst)
+}
+
 // Test with BackupDir set
 func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
 	r := fstest.NewRun(t)