Repository: rclone/rclone
Pull request #7001 (2023-05-11 16:12:46)

azblob: support azure workload identity authentication
#### What is the purpose of this change?

Allow using azure workload identity authentication for azblob backend. Available through the standard environment authentication chain when azblob's `env_auth` is set to `true`.

#### Was the change discussed in an issue or in the forum before?

No.

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [ ] I have added tests for all changes in this PR if appropriate.
- [ ] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [ ] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/backend/combine/combine.go b/backend/combine/combine.go
index 2e1ad2a7b..f83a3c486 100644
--- a/backend/combine/combine.go
+++ b/backend/combine/combine.go
@@ -289,6 +289,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		}
 	}
 
+	// Enable CleanUp when any upstreams support it
+	if features.CleanUp == nil {
+		for _, u := range f.upstreams {
+			if u.f.Features().CleanUp != nil {
+				features.CleanUp = f.CleanUp
+				break
+			}
+		}
+	}
+
 	// Enable ChangeNotify when any upstreams support it
 	if features.ChangeNotify == nil {
 		for _, u := range f.upstreams {
@@ -299,6 +309,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		}
 	}
 
+	// show that we wrap other backends
+	features.Overlay = true
+
 	f.features = features
 
 	// Get common intersection of hashes
@@ -887,6 +900,100 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 	})
 }
 
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	u, uRemote, err := f.findUpstream(remote)
+	if err != nil {
+		return "", err
+	}
+	do := u.f.Features().PublicLink
+	if do == nil {
+		return "", fs.ErrorNotImplemented
+	}
+	return do(ctx, uRemote, expire, unlink)
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+//
+// May create duplicates or return errors if src already
+// exists.
+func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	srcPath := src.Remote()
+	u, uRemote, err := f.findUpstream(srcPath)
+	if err != nil {
+		return nil, err
+	}
+	do := u.f.Features().PutUnchecked
+	if do == nil {
+		return nil, fs.ErrorNotImplemented
+	}
+	uSrc := fs.NewOverrideRemote(src, uRemote)
+	return do(ctx, in, uSrc, options...)
+}
+
+// MergeDirs merges the contents of all the directories passed
+// in into the first one and rmdirs the other directories.
+func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
+	if len(dirs) == 0 {
+		return nil
+	}
+	var (
+		u     *upstream
+		uDirs []fs.Directory
+	)
+	for _, dir := range dirs {
+		uNew, uDir, err := f.findUpstream(dir.Remote())
+		if err != nil {
+			return err
+		}
+		if u == nil {
+			u = uNew
+		} else if u != uNew {
+			return fmt.Errorf("can't merge dirctories from different upstreams")
+		}
+		uDirs = append(uDirs, fs.NewOverrideDirectory(dir, uDir))
+	}
+	do := u.f.Features().MergeDirs
+	if do == nil {
+		return fs.ErrorNotImplemented
+	}
+	return do(ctx, uDirs)
+}
+
+// CleanUp the trash in the Fs
+//
+// Implement this if you have a way of emptying the trash or
+// otherwise cleaning up old versions of files.
+func (f *Fs) CleanUp(ctx context.Context) error {
+	return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
+		if do := u.f.Features().CleanUp; do != nil {
+			return do(ctx)
+		}
+		return nil
+	})
+}
+
+// OpenWriterAt opens with a handle for random access writes
+//
+// Pass in the remote desired and the size if known.
+//
+// It truncates any existing object
+func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
+	u, uRemote, err := f.findUpstream(remote)
+	if err != nil {
+		return nil, err
+	}
+	do := u.f.Features().OpenWriterAt
+	if do == nil {
+		return nil, fs.ErrorNotImplemented
+	}
+	return do(ctx, uRemote, size)
+}
+
 // Object describes a wrapped Object
 //
 // This is a wrapped Object which knows its path prefix
@@ -988,5 +1095,10 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
+	_ fs.PutUncheckeder  = (*Fs)(nil)
+	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.CleanUpper      = (*Fs)(nil)
+	_ fs.OpenWriterAter  = (*Fs)(nil)
 	_ fs.FullObject      = (*Object)(nil)
 )
diff --git a/backend/pikpak/api/types.go b/backend/pikpak/api/types.go
index e5a3ee0f5..fab6951eb 100644
--- a/backend/pikpak/api/types.go
+++ b/backend/pikpak/api/types.go
@@ -227,9 +227,10 @@ type Media struct {
 		Duration   int64  `json:"duration,omitempty"`
 		BitRate    int    `json:"bit_rate,omitempty"`
 		FrameRate  int    `json:"frame_rate,omitempty"`
-		VideoCodec string `json:"video_codec,omitempty"`
-		AudioCodec string `json:"audio_codec,omitempty"`
-		VideoType  string `json:"video_type,omitempty"`
+		VideoCodec string `json:"video_codec,omitempty"` // "h264", "hevc"
+		AudioCodec string `json:"audio_codec,omitempty"` // "pcm_bluray", "aac"
+		VideoType  string `json:"video_type,omitempty"`  // "mpegts"
+		HdrType    string `json:"hdr_type,omitempty"`
 	} `json:"video,omitempty"`
 	Link           *Link         `json:"link,omitempty"`
 	NeedMoreQuota  bool          `json:"need_more_quota,omitempty"`
diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.go
index 616f62607..1f9790a1c 100644
--- a/backend/pikpak/pikpak.go
+++ b/backend/pikpak/pikpak.go
@@ -1411,6 +1411,16 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 
 // ------------------------------------------------------------
 
+// parseFileID gets fid parameter from url query
+func parseFileID(s string) string {
+	if u, err := url.Parse(s); err == nil {
+		if q, err := url.ParseQuery(u.RawQuery); err == nil {
+			return q.Get("fid")
+		}
+	}
+	return ""
+}
+
 // setMetaData sets the metadata from info
 func (o *Object) setMetaData(info *api.File) (err error) {
 	if info.Kind == api.KindOfFolder {
@@ -1432,10 +1442,18 @@ func (o *Object) setMetaData(info *api.File) (err error) {
 	o.md5sum = info.Md5Checksum
 	if info.Links.ApplicationOctetStream != nil {
 		o.link = info.Links.ApplicationOctetStream
-	}
-	if len(info.Medias) > 0 && info.Medias[0].Link != nil {
-		fs.Debugf(o, "Using a media link")
-		o.link = info.Medias[0].Link
+		if fid := parseFileID(o.link.URL); fid != "" {
+			for mid, media := range info.Medias {
+				if media.Link == nil {
+					continue
+				}
+				if mfid := parseFileID(media.Link.URL); fid == mfid {
+					fs.Debugf(o, "Using a media link from Medias[%d]", mid)
+					o.link = media.Link
+					break
+				}
+			}
+		}
 	}
 	return nil
 }
diff --git a/backend/s3/v2sign.go b/backend/s3/v2sign.go
index d31cbeb71..12e41f0b6 100644
--- a/backend/s3/v2sign.go
+++ b/backend/s3/v2sign.go
@@ -14,6 +14,7 @@ import (
 
 // URL parameters that need to be added to the signature
 var s3ParamsToSign = map[string]struct{}{
+	"delete":                       {},
 	"acl":                          {},
 	"location":                     {},
 	"logging":                      {},
diff --git a/backend/union/union.go b/backend/union/union.go
index 1b3c0f4db..268860621 100644
--- a/backend/union/union.go
+++ b/backend/union/union.go
@@ -801,6 +801,24 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 	return errs.Err()
 }
 
+// CleanUp the trash in the Fs
+//
+// Implement this if you have a way of emptying the trash or
+// otherwise cleaning up old versions of files.
+func (f *Fs) CleanUp(ctx context.Context) error {
+	errs := Errors(make([]error, len(f.upstreams)))
+	multithread(len(f.upstreams), func(i int) {
+		u := f.upstreams[i]
+		if do := u.Features().CleanUp; do != nil {
+			err := do(ctx)
+			if err != nil {
+				errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
+			}
+		}
+	})
+	return errs.Err()
+}
+
 // NewFs constructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
@@ -914,6 +932,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 	}
 
+	// show that we wrap other backends
+	features.Overlay = true
+
 	f.features = features
 
 	// Get common intersection of hashes
@@ -960,4 +981,5 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Shutdowner      = (*Fs)(nil)
+	_ fs.CleanUpper      = (*Fs)(nil)
 )
diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md
index e402c546a..6520bcf9b 100644
--- a/docs/content/azureblob.md
+++ b/docs/content/azureblob.md
@@ -162,6 +162,12 @@ It reads configuration from these variables, in the following order:
     - `AZURE_CLIENT_ID`: client ID of the application the user will authenticate to
     - `AZURE_USERNAME`: a username (usually an email address)
     - `AZURE_PASSWORD`: the user's password
+4. Workload Identity
+    - `AZURE_TENANT_ID`: Tenant to authenticate in.
+    - `AZURE_CLIENT_ID`: Client ID of the application the user will authenticate to.
+    - `AZURE_FEDERATED_TOKEN_FILE`: Path to projected service account token file.
+    - `AZURE_AUTHORITY_HOST`: Authority of an Azure Active Directory endpoint (default: login.microsoftonline.com).
+
 
 ##### Env Auth: 2. Managed Service Identity Credentials
 
diff --git a/fs/features.go b/fs/features.go
index 2ba54abe2..4c3612285 100644
--- a/fs/features.go
+++ b/fs/features.go
@@ -32,6 +32,7 @@ type Features struct {
 	FilterAware             bool // can make use of filters if provided for listing
 	PartialUploads          bool // uploaded file can appear incomplete on the fs while it's being uploaded
 	NoMultiThreading        bool // set if can't have multiplethreads on one download open
+	Overlay                 bool // this wraps one or more backends to add functionality
 
 	// Purge all files in the directory specified
 	//
@@ -271,6 +272,7 @@ func (ft *Features) Fill(ctx context.Context, f Fs) *Features {
 	if do, ok := f.(Wrapper); ok {
 		ft.WrapFs = do.WrapFs
 		ft.SetWrapper = do.SetWrapper
+		ft.Overlay = true // if implement UnWrap then must be an Overlay
 	}
 	if do, ok := f.(DirCacheFlusher); ok {
 		ft.DirCacheFlush = do.DirCacheFlush
@@ -339,6 +341,9 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
 	ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
 	ft.SlowHash = ft.SlowHash && mask.SlowHash
 	ft.FilterAware = ft.FilterAware && mask.FilterAware
+	ft.PartialUploads = ft.PartialUploads && mask.PartialUploads
+	ft.NoMultiThreading = ft.NoMultiThreading && mask.NoMultiThreading
+	// ft.Overlay = ft.Overlay && mask.Overlay don't propagate Overlay
 
 	if mask.Purge == nil {
 		ft.Purge = nil
diff --git a/fs/override_dir.go b/fs/override_dir.go
new file mode 100644
index 000000000..a20651155
--- /dev/null
+++ b/fs/override_dir.go
@@ -0,0 +1,34 @@
+package fs
+
+// OverrideDirectory is a wrapper to override the Remote for an
+// Directory
+type OverrideDirectory struct {
+	Directory
+	remote string
+}
+
+// NewOverrideDirectory returns an OverrideDirectoryObject which will
+// return the remote specified
+func NewOverrideDirectory(oi Directory, remote string) *OverrideDirectory {
+	// re-wrap an OverrideDirectory
+	if or, ok := oi.(*OverrideDirectory); ok {
+		return &OverrideDirectory{
+			Directory: or.Directory,
+			remote:    remote,
+		}
+	}
+	return &OverrideDirectory{
+		Directory: oi,
+		remote:    remote,
+	}
+}
+
+// Remote returns the overridden remote name
+func (o *OverrideDirectory) Remote() string {
+	return o.remote
+}
+
+// String returns the overridden remote name
+func (o *OverrideDirectory) String() string {
+	return o.remote
+}
diff --git a/go.mod b/go.mod
index 2851dde02..cee2d6453 100644
--- a/go.mod
+++ b/go.mod
@@ -4,8 +4,8 @@ go 1.18
 
 require (
 	bazil.org/fuse v0.0.0-20221209211307-2abb8038c751
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd
@@ -78,12 +78,12 @@ require (
 require (
 	cloud.google.com/go/compute v1.19.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cloudflare/circl v1.1.0 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 7174e741f..c7a505817 100644
--- a/go.sum
+++ b/go.sum
@@ -41,16 +41,24 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE=
@@ -107,8 +115,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY=
 github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
+github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
+github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/colinmarc/hdfs/v2 v2.3.0 h1:tMxOjXn6+7iPUlxAyup9Ha2hnmLe3Sv5DM2qqbSQ2VY=
 github.com/colinmarc/hdfs/v2 v2.3.0/go.mod h1:nsyY1uyQOomU34KVQk9Qb/lDJobN1MQ/9WS6IqcVZno=


Test patch:
diff --git a/backend/combine/combine_test.go b/backend/combine/combine_test.go
index 46b49bcf0..0a0d357dd 100644
--- a/backend/combine/combine_test.go
+++ b/backend/combine/combine_test.go
@@ -10,6 +10,11 @@ import (
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
+var (
+	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect"}
+	unimplementableObjectMethods = []string{}
+)
+
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	if *fstest.RemoteName == "" {
@@ -17,8 +22,8 @@ func TestIntegration(t *testing.T) {
 	}
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   *fstest.RemoteName,
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 	})
 }
 
@@ -35,7 +40,9 @@ func TestLocal(t *testing.T) {
 			{Name: name, Key: "type", Value: "combine"},
 			{Name: name, Key: "upstreams", Value: upstreams},
 		},
-		QuickTestOK: true,
+		QuickTestOK:                  true,
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 	})
 }
 
@@ -51,7 +58,9 @@ func TestMemory(t *testing.T) {
 			{Name: name, Key: "type", Value: "combine"},
 			{Name: name, Key: "upstreams", Value: upstreams},
 		},
-		QuickTestOK: true,
+		QuickTestOK:                  true,
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 	})
 }
 
@@ -68,6 +77,8 @@ func TestMixed(t *testing.T) {
 			{Name: name, Key: "type", Value: "combine"},
 			{Name: name, Key: "upstreams", Value: upstreams},
 		},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 	})
 }
 
diff --git a/backend/union/union_test.go b/backend/union/union_test.go
index f5a9948f6..c9482c56c 100644
--- a/backend/union/union_test.go
+++ b/backend/union/union_test.go
@@ -11,6 +11,11 @@ import (
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
+var (
+	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "PublicLink", "PutUnchecked", "MergeDirs", "OpenWriterAt"}
+	unimplementableObjectMethods = []string{}
+)
+
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	if *fstest.RemoteName == "" {
@@ -18,8 +23,8 @@ func TestIntegration(t *testing.T) {
 	}
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   *fstest.RemoteName,
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 	})
 }
 
@@ -39,8 +44,8 @@ func TestStandard(t *testing.T) {
 			{Name: name, Key: "create_policy", Value: "epmfs"},
 			{Name: name, Key: "search_policy", Value: "ff"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 		QuickTestOK:                  true,
 	})
 }
@@ -61,8 +66,8 @@ func TestRO(t *testing.T) {
 			{Name: name, Key: "create_policy", Value: "epmfs"},
 			{Name: name, Key: "search_policy", Value: "ff"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 		QuickTestOK:                  true,
 	})
 }
@@ -83,8 +88,8 @@ func TestNC(t *testing.T) {
 			{Name: name, Key: "create_policy", Value: "epmfs"},
 			{Name: name, Key: "search_policy", Value: "ff"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 		QuickTestOK:                  true,
 	})
 }
@@ -105,8 +110,8 @@ func TestPolicy1(t *testing.T) {
 			{Name: name, Key: "create_policy", Value: "lus"},
 			{Name: name, Key: "search_policy", Value: "all"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 		QuickTestOK:                  true,
 	})
 }
@@ -127,8 +132,8 @@ func TestPolicy2(t *testing.T) {
 			{Name: name, Key: "create_policy", Value: "rand"},
 			{Name: name, Key: "search_policy", Value: "ff"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 		QuickTestOK:                  true,
 	})
 }
@@ -149,8 +154,8 @@ func TestPolicy3(t *testing.T) {
 			{Name: name, Key: "create_policy", Value: "all"},
 			{Name: name, Key: "search_policy", Value: "all"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
+		UnimplementableFsMethods:     unimplementableFsMethods,
+		UnimplementableObjectMethods: unimplementableObjectMethods,
 		QuickTestOK:                  true,
 	})
 }
diff --git a/fs/override_dir_test.go b/fs/override_dir_test.go
new file mode 100644
index 000000000..0e23e1b55
--- /dev/null
+++ b/fs/override_dir_test.go
@@ -0,0 +1,4 @@
+package fs
+
+// Check interfaces satisfied
+var _ Directory = (*OverrideDirectory)(nil)
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go
index 46a6ff4b5..ddd20c97a 100644
--- a/fstest/fstests/fstests.go
+++ b/fstest/fstests/fstests.go
@@ -460,7 +460,7 @@ func Run(t *testing.T, opt *Opt) {
 			t.Skip("Skipping FsCheckWrap on this Fs")
 		}
 		ft := new(fs.Features).Fill(ctx, f)
-		if ft.UnWrap == nil {
+		if ft.UnWrap == nil && !f.Features().Overlay {
 			t.Skip("Not a wrapping Fs")
 		}
 		v := reflect.ValueOf(ft).Elem()