Repository: rclone/rclone
Pull request #6983 (2023-04-29 12:54:43)

mount2: fixes & updates
#### What is the purpose of this change?

Various fixes & updates for mount2

#### Was the change discussed in an issue or in the forum before?

N/A

#### Checklist

- [X] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [X] I have added tests for all changes in this PR if appropriate.
- [X] I have added documentation for the changes if appropriate.
- [X] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [X] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/cmd/mount/handle.go b/cmd/mount/handle.go
index 7b7b6c010..c3293acc3 100644
--- a/cmd/mount/handle.go
+++ b/cmd/mount/handle.go
@@ -25,15 +25,13 @@ var _ fusefs.HandleReader = (*FileHandle)(nil)
 func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) (err error) {
 	var n int
 	defer log.Trace(fh, "len=%d, offset=%d", req.Size, req.Offset)("read=%d, err=%v", &n, &err)
-	data := make([]byte, req.Size)
+	data := resp.Data[:req.Size]
 	n, err = fh.Handle.ReadAt(data, req.Offset)
+	resp.Data = data[:n]
 	if err == io.EOF {
 		err = nil
-	} else if err != nil {
-		return translateError(err)
 	}
-	resp.Data = data[:n]
-	return nil
+	return translateError(err)
 }
 
 // Check interface satisfied
diff --git a/cmd/mount2/mount.go b/cmd/mount2/mount.go
index 152f3a89e..811cab235 100644
--- a/cmd/mount2/mount.go
+++ b/cmd/mount2/mount.go
@@ -26,12 +26,14 @@ func init() {
 // man mount.fuse for more info and note the -o flag for other options
 func mountOptions(fsys *FS, f fs.Fs, opt *mountlib.Options) (mountOpts *fuse.MountOptions) {
 	mountOpts = &fuse.MountOptions{
-		AllowOther:    fsys.opt.AllowOther,
-		FsName:        opt.DeviceName,
-		Name:          "rclone",
-		DisableXAttrs: true,
-		Debug:         fsys.opt.DebugFUSE,
-		MaxReadAhead:  int(fsys.opt.MaxReadAhead),
+		AllowOther:         fsys.opt.AllowOther,
+		FsName:             opt.DeviceName,
+		Name:               "rclone",
+		DisableXAttrs:      true,
+		Debug:              fsys.opt.DebugFUSE,
+		MaxReadAhead:       int(fsys.opt.MaxReadAhead),
+		MaxWrite:           1024 * 1024, // Linux v4.20+ caps requests at 1 MiB
+		DisableReadDirPlus: true,
 
 		// RememberInodes: true,
 		// SingleThreaded: true,
@@ -47,12 +49,42 @@ func mountOptions(fsys *FS, f fs.Fs, opt *mountlib.Options) (mountOpts *fuse.Mou
 			// async I/O.  Concurrency for synchronous I/O is not limited.
 			MaxBackground int
 
-			// Write size to use.  If 0, use default. This number is
-			// capped at the kernel maximum.
+			// MaxWrite is the max size for read and write requests. If 0, use
+			// go-fuse default (currently 64 kiB).
+			// This number is internally capped at MAX_KERNEL_WRITE (higher values don't make
+			// sense).
+			//
+			// Non-direct-io reads are mostly served via kernel readahead, which is
+			// additionally subject to the MaxReadAhead limit.
+			//
+			// Implementation notes:
+			//
+			// There's four values the Linux kernel looks at when deciding the request size:
+			// * MaxWrite, passed via InitOut.MaxWrite. Limits the WRITE size.
+			// * max_read, passed via a string mount option. Limits the READ size.
+			//   go-fuse sets max_read equal to MaxWrite.
+			//   You can see the current max_read value in /proc/self/mounts .
+			// * MaxPages, passed via InitOut.MaxPages. In Linux 4.20 and later, the value
+			//   can go up to 1 MiB and go-fuse calculates the MaxPages value acc.
+			//   to MaxWrite, rounding up.
+			//   On older kernels, the value is fixed at 128 kiB and the
+			//   passed value is ignored. No request can be larger than MaxPages, so
+			//   READ and WRITE are effectively capped at MaxPages.
+			// * MaxReadAhead, passed via InitOut.MaxReadAhead.
 			MaxWrite int
 
-			// Max read ahead to use.  If 0, use default. This number is
-			// capped at the kernel maximum.
+			// MaxReadAhead is the max read ahead size to use. It controls how much data the
+			// kernel reads in advance to satisfy future read requests from applications.
+			// How much exactly is subject to clever heuristics in the kernel
+			// (see https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/mm/readahead.c?h=v6.2-rc5#n375
+			// if you are brave) and hence also depends on the kernel version.
+			//
+			// If 0, use kernel default. This number is capped at the kernel maximum
+			// (128 kiB on Linux) and cannot be larger than MaxWrite.
+			//
+			// MaxReadAhead only affects buffered reads (=non-direct-io), but even then, the
+			// kernel can and does send larger reads to satisfy read reqests from applications
+			// (up to MaxWrite or VM_READAHEAD_PAGES=128 kiB, whichever is less).
 			MaxReadAhead int
 
 			// If IgnoreSecurityLabels is set, all security related xattr
@@ -87,9 +119,19 @@ func mountOptions(fsys *FS, f fs.Fs, opt *mountlib.Options) (mountOpts *fuse.Mou
 			// you must implement the GetLk/SetLk/SetLkw methods.
 			EnableLocks bool
 
+			// If set, the kernel caches all Readlink return values. The
+			// filesystem must use content notification to force the
+			// kernel to issue a new Readlink call.
+			EnableSymlinkCaching bool
+
 			// If set, ask kernel not to do automatic data cache invalidation.
 			// The filesystem is fully responsible for invalidating data cache.
 			ExplicitDataCacheControl bool
+
+			// Disable ReadDirPlus capability so ReadDir is used instead. Simple
+			// directory queries (i.e. 'ls' without '-l') can be faster with
+			// ReadDir, as no per-file stat calls are needed
+			DisableReadDirPlus bool
 		*/
 
 	}
@@ -176,8 +218,8 @@ func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error
 		MountOptions: *mountOpts,
 		EntryTimeout: &opt.AttrTimeout,
 		AttrTimeout:  &opt.AttrTimeout,
-		// UID
-		// GID
+		GID:          VFS.Opt.GID,
+		UID:          VFS.Opt.UID,
 	}
 
 	root, err := fsys.Root()
diff --git a/cmd/mount2/node.go b/cmd/mount2/node.go
index fae416d64..0ba1aded4 100644
--- a/cmd/mount2/node.go
+++ b/cmd/mount2/node.go
@@ -85,17 +85,16 @@ func (n *Node) lookupVfsNodeInDir(leaf string) (vfsNode vfs.Node, errno syscall.
 // will not work.
 func (n *Node) Statfs(ctx context.Context, out *fuse.StatfsOut) syscall.Errno {
 	defer log.Trace(n, "")("out=%+v", &out)
-	out = new(fuse.StatfsOut)
 	const blockSize = 4096
-	const fsBlocks = (1 << 50) / blockSize
-	out.Blocks = fsBlocks  // Total data blocks in file system.
-	out.Bfree = fsBlocks   // Free blocks in file system.
-	out.Bavail = fsBlocks  // Free blocks in file system if you're not root.
-	out.Files = 1e9        // Total files in file system.
-	out.Ffree = 1e9        // Free files in file system.
-	out.Bsize = blockSize  // Block size
-	out.NameLen = 255      // Maximum file name length?
-	out.Frsize = blockSize // Fragment size, smallest addressable data size in the file system.
+	total, _, free := n.fsys.VFS.Statfs()
+	out.Blocks = uint64(total) / blockSize // Total data blocks in file system.
+	out.Bfree = uint64(free) / blockSize   // Free blocks in file system.
+	out.Bavail = out.Bfree                 // Free blocks in file system if you're not root.
+	out.Files = 1e9                        // Total files in file system.
+	out.Ffree = 1e9                        // Free files in file system.
+	out.Bsize = blockSize                  // Block size
+	out.NameLen = 255                      // Maximum file name length?
+	out.Frsize = blockSize                 // Fragment size, smallest addressable data size in the file system.
 	mountlib.ClipBlocks(&out.Blocks)
 	mountlib.ClipBlocks(&out.Bfree)
 	mountlib.ClipBlocks(&out.Bavail)
@@ -405,3 +404,40 @@ func (n *Node) Rename(ctx context.Context, oldName string, newParent fusefs.Inod
 }
 
 var _ = (fusefs.NodeRenamer)((*Node)(nil))
+
+// Getxattr should read data for the given attribute into
+// `dest` and return the number of bytes. If `dest` is too
+// small, it should return ERANGE and the size of the attribute.
+// If not defined, Getxattr will return ENOATTR.
+func (n *Node) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) {
+	return 0, syscall.ENOSYS // we never implement this
+}
+
+var _ fusefs.NodeGetxattrer = (*Node)(nil)
+
+// Setxattr should store data for the given attribute.  See
+// setxattr(2) for information about flags.
+// If not defined, Setxattr will return ENOATTR.
+func (n *Node) Setxattr(ctx context.Context, attr string, data []byte, flags uint32) syscall.Errno {
+	return syscall.ENOSYS // we never implement this
+}
+
+var _ fusefs.NodeSetxattrer = (*Node)(nil)
+
+// Removexattr should delete the given attribute.
+// If not defined, Removexattr will return ENOATTR.
+func (n *Node) Removexattr(ctx context.Context, attr string) syscall.Errno {
+	return syscall.ENOSYS // we never implement this
+}
+
+var _ fusefs.NodeRemovexattrer = (*Node)(nil)
+
+// Listxattr should read all attributes (null terminated) into
+// `dest`. If the `dest` buffer is too small, it should return ERANGE
+// and the correct size.  If not defined, return an empty list and
+// success.
+func (n *Node) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errno) {
+	return 0, syscall.ENOSYS // we never implement this
+}
+
+var _ fusefs.NodeListxattrer = (*Node)(nil)
diff --git a/go.mod b/go.mod
index da6987c4f..988a2b420 100644
--- a/go.mod
+++ b/go.mod
@@ -28,7 +28,7 @@ require (
 	github.com/gdamore/tcell/v2 v2.6.0
 	github.com/go-chi/chi/v5 v5.0.8
 	github.com/google/uuid v1.3.0
-	github.com/hanwen/go-fuse/v2 v2.2.0
+	github.com/hanwen/go-fuse/v2 v2.2.1-0.20230410213758-80c1c8221982
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/iguanesolutions/go-systemd/v5 v5.1.1
 	github.com/jcmturner/gokrb5/v8 v8.4.4
diff --git a/go.sum b/go.sum
index d7a4e3749..a39ae52de 100644
--- a/go.sum
+++ b/go.sum
@@ -248,8 +248,8 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=
 github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
-github.com/hanwen/go-fuse/v2 v2.2.0 h1:jo5QZYmBLNcl9ovypWaQ5yXMSSV+Ch68xoC3rtZvvBM=
-github.com/hanwen/go-fuse/v2 v2.2.0/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc=
+github.com/hanwen/go-fuse/v2 v2.2.1-0.20230410213758-80c1c8221982 h1:2620K4xZZUW81Nef9jqNJNS1UxAO6Fsb//FVDSC3KHw=
+github.com/hanwen/go-fuse/v2 v2.2.1-0.20230410213758-80c1c8221982/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
@@ -345,6 +345,8 @@ github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tB
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
+github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=


Test patch:
diff --git a/cmd/mount2/mount_test.go b/cmd/mount2/mount_test.go
index c6572a983..e90632f02 100644
--- a/cmd/mount2/mount_test.go
+++ b/cmd/mount2/mount_test.go
@@ -1,16 +1,14 @@
-//go:build linux || (darwin && amd64)
-// +build linux darwin,amd64
+//go:build linux
+// +build linux
 
 package mount2
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/vfs/vfstest"
 )
 
 func TestMount(t *testing.T) {
-	testy.SkipUnreliable(t)
 	vfstest.RunTests(t, false, mount)
 }