Repository: rclone/rclone
Pull request #6140 (2022-05-02 12:21:03)

Accept nedev arg
#### What is the purpose of this change?

Do not trigger an error when parsing mount options starting with an underscore.

#### Was the change discussed in an issue or in the forum before?

This make possible the example given in #5594. 

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [x] I have added tests for all changes in this PR if appropriate.
- [ ] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)


ncw: I think this fixes this issue: https://github.com/rclone/rclone/issues/5808

Can you take a look through that issue and see if you think so too, and if so add a `Fixes #5808` to the commit message.

Patch:
diff --git a/README.md b/README.md
index 1f3ca3dfb..0235323b5 100644
--- a/README.md
+++ b/README.md
@@ -29,6 +29,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
   * China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
+  * Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
   * Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go
index ce994c23c..ce021e35f 100644
--- a/backend/internetarchive/internetarchive.go
+++ b/backend/internetarchive/internetarchive.go
@@ -28,6 +28,7 @@ import (
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -129,6 +130,7 @@ type IAFile struct {
 	// Source     string `json:"source"`
 	Mtime       string          `json:"mtime"`
 	RcloneMtime json.RawMessage `json:"rclone-mtime"`
+	UpdateTrack json.RawMessage `json:"rclone-update-track"`
 	Size        string          `json:"size"`
 	Md5         string          `json:"md5"`
 	Crc32       string          `json:"crc32"`
@@ -294,7 +296,7 @@ func (o *Object) Storable() bool {
 	return true
 }
 
-// SetModTime is not supported
+// SetModTime sets modTime on a particular file
 func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	bucket, reqDir := o.split()
 	if bucket == "" {
@@ -483,6 +485,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		return nil, fs.ErrorCantCopy
 	}
 
+	updateTracker := random.String(32)
 	headers := map[string]string{
 		"x-archive-auto-make-bucket": "1",
 		"x-archive-queue-derive":     "0",
@@ -494,7 +497,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		"x-archive-filemeta-crc32":   srcObj.crc32,
 		"x-archive-filemeta-size":    fmt.Sprint(srcObj.size),
 		// add this too for sure
-		"x-archive-filemeta-rclone-mtime": srcObj.modTime.Format(time.RFC3339Nano),
+		"x-archive-filemeta-rclone-mtime":        srcObj.modTime.Format(time.RFC3339Nano),
+		"x-archive-filemeta-rclone-update-track": updateTracker,
 	}
 
 	// make a PUT request at (IAS3)/:item/:path without body
@@ -515,7 +519,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 
 	// we can't update/find metadata here as IA will also
 	// queue server-side copy as well as upload/delete.
-	return f.waitFileUpload(ctx, trimPathPrefix(path.Join(dstBucket, dstPath), f.root, f.opt.Enc), f.getHashes(ctx, src), srcObj.size)
+	return f.waitFileUpload(ctx, trimPathPrefix(path.Join(dstBucket, dstPath), f.root, f.opt.Enc), updateTracker, srcObj.size)
 }
 
 // ListR lists the objects and directories of the Fs starting
@@ -660,12 +664,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	bucket, bucketPath := o.split()
 	modTime := src.ModTime(ctx)
 	size := src.Size()
+	updateTracker := random.String(32)
 
 	// Set the mtime in the metadata
 	// internetarchive backend builds at header level as IAS3 has extension outside X-Amz-
 	headers := map[string]string{
 		// https://github.com/jjjake/internetarchive/blob/2456376533251df9d05e0a14d796ec1ced4959f5/internetarchive/iarequest.py#L158
-		"x-amz-filemeta-rclone-mtime": modTime.Format(time.RFC3339Nano),
+		"x-amz-filemeta-rclone-mtime":        modTime.Format(time.RFC3339Nano),
+		"x-amz-filemeta-rclone-update-track": updateTracker,
 
 		// we add some more headers for intuitive actions
 		"x-amz-auto-make-bucket":     "1",    // create an item if does not exist, do nothing if already
@@ -712,7 +718,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)
 	var newObj *Object
 	if err == nil {
-		newObj, err = o.fs.waitFileUpload(ctx, o.remote, o.fs.getHashes(ctx, src), size)
+		newObj, err = o.fs.waitFileUpload(ctx, o.remote, updateTracker, size)
 	} else {
 		newObj = &Object{}
 	}
@@ -782,18 +788,6 @@ func (o *Object) split() (bucket, bucketPath string) {
 	return o.fs.split(o.remote)
 }
 
-func (f *Fs) getHashes(ctx context.Context, src fs.ObjectInfo) map[hash.Type]string {
-	hashMap := map[hash.Type]string{}
-	for _, ty := range f.Hashes().Array() {
-		sum, err := src.Hash(ctx, ty)
-		if err != nil || sum == "" {
-			continue
-		}
-		hashMap[ty] = sum
-	}
-	return hashMap
-}
-
 func (f *Fs) requestMetadata(ctx context.Context, bucket string) (result MetadataResponse, err error) {
 	var resp *http.Response
 	// make a GET request to (frontend)/metadata/:item/
@@ -852,7 +846,7 @@ func (f *Fs) listAllUnconstrained(ctx context.Context, bucket string) (entries f
 	return entries, nil
 }
 
-func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[hash.Type]string, newSize int64) (ret *Object, err error) {
+func (f *Fs) waitFileUpload(ctx context.Context, reqPath, tracker string, newSize int64) (ret *Object, err error) {
 	bucket, bucketPath := f.split(reqPath)
 
 	ret = &Object{
@@ -869,6 +863,10 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 			ret2, ok := ret2.(*Object)
 			if ok {
 				ret = ret2
+				ret.crc32 = ""
+				ret.md5 = ""
+				ret.sha1 = ""
+				ret.size = -1
 			}
 		}
 		return ret, nil
@@ -881,9 +879,6 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 	go func() {
 		isFirstTime := true
 		existed := false
-		oldMtime := ""
-		oldCrc32 := ""
-		unreliablePassCount := 0
 		for {
 			if !isFirstTime {
 				// depending on the queue, it takes time
@@ -908,10 +903,6 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 			if isFirstTime {
 				isFirstTime = false
 				existed = iaFile != nil
-				if iaFile != nil {
-					oldMtime = iaFile.Mtime
-					oldCrc32 = iaFile.Crc32
-				}
 			}
 			if iaFile == nil {
 				continue
@@ -925,38 +916,20 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 				return
 			}
 
-			hashMatched := true
-			for tt, sum := range newHashes {
-				if tt == hash.MD5 && !hash.Equals(iaFile.Md5, sum) {
-					hashMatched = false
-					break
-				}
-				if tt == hash.SHA1 && !hash.Equals(iaFile.Sha1, sum) {
-					hashMatched = false
-					break
-				}
-				if tt == hash.CRC32 && !hash.Equals(iaFile.Crc32, sum) {
-					hashMatched = false
+			fileTrackers, _ := listOrString(iaFile.UpdateTrack)
+			trackerMatch := false
+			for _, v := range fileTrackers {
+				if v == tracker {
+					trackerMatch = true
 					break
 				}
 			}
-			if !hashMatched {
+			if !trackerMatch {
 				continue
 			}
 			if !compareSize(parseSize(iaFile.Size), newSize) {
 				continue
 			}
-			if hash.Equals(oldCrc32, iaFile.Crc32) && unreliablePassCount < 60 {
-				// the following two are based on a sort of "bad" assumption;
-				// what if the file is updated immediately, before polling?
-				// by limiting hits of these tests, avoid infinite loop
-				unreliablePassCount++
-				continue
-			}
-			if hash.Equals(iaFile.Mtime, oldMtime) && unreliablePassCount < 60 {
-				unreliablePassCount++
-				continue
-			}
 
 			// voila!
 			retC <- struct {
@@ -1036,20 +1009,24 @@ func makeValidObject2(f *Fs, file IAFile, bucket string) *Object {
 	return makeValidObject(f, trimPathPrefix(path.Join(bucket, file.Name), f.root, f.opt.Enc), file, mtimeTime, size)
 }
 
-func (file IAFile) parseMtime() (mtime time.Time) {
-	// method 1: use metadata added by rclone
-	var rmArray []string
+func listOrString(jm json.RawMessage) (rmArray []string, err error) {
 	// rclone-metadata can be an array or string
 	// try to deserialize it as array first
-	err := json.Unmarshal(file.RcloneMtime, &rmArray)
+	err = json.Unmarshal(jm, &rmArray)
 	if err != nil {
 		// if not, it's a string
 		dst := new(string)
-		err = json.Unmarshal(file.RcloneMtime, dst)
+		err = json.Unmarshal(jm, dst)
 		if err == nil {
 			rmArray = []string{*dst}
 		}
 	}
+	return
+}
+
+func (file IAFile) parseMtime() (mtime time.Time) {
+	// method 1: use metadata added by rclone
+	rmArray, err := listOrString(file.RcloneMtime)
 	// let's take the first value we can deserialize
 	for _, value := range rmArray {
 		mtime, err = time.Parse(time.RFC3339Nano, value)
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index afc9b36f5..d8ba92724 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -32,6 +32,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/ncw/swift/v2"
 	"github.com/rclone/rclone/fs"
@@ -58,7 +59,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Options: []fs.Option{{
@@ -78,6 +79,9 @@ func init() {
 			}, {
 				Value: "ChinaMobile",
 				Help:  "China Mobile Ecloud Elastic Object Storage (EOS)",
+			}, {
+				Value: "Cloudflare",
+				Help:  "Cloudflare R2 Storage",
 			}, {
 				Value: "ArvanCloud",
 				Help:  "Arvan Cloud Object Storage (AOS)",
@@ -297,10 +301,18 @@ func init() {
 				Value: "fr-par",
 				Help:  "Paris, France",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region to connect to.",
+			Provider: "Cloudflare",
+			Examples: []fs.OptionExample{{
+				Value: "auto",
+				Help:  "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
+			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -1270,7 +1282,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
+			Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1358,7 +1370,7 @@ isn't set then "acl" is used instead.`,
 		}, {
 			Name:     "server_side_encryption",
 			Help:     "The server-side encryption algorithm used when storing this object in S3.",
-			Provider: "AWS,Ceph,ChinaMobile,ArvanCloud,Minio",
+			Provider: "AWS,Ceph,ChinaMobile,Minio",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "None",
@@ -1373,7 +1385,7 @@ isn't set then "acl" is used instead.`,
 		}, {
 			Name:     "sse_customer_algorithm",
 			Help:     "If using SSE-C, the server-side encryption algorithm used when storing this object in S3.",
-			Provider: "AWS,Ceph,ChinaMobile,ArvanCloud,Minio",
+			Provider: "AWS,Ceph,ChinaMobile,Minio",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -1385,7 +1397,7 @@ isn't set then "acl" is used instead.`,
 		}, {
 			Name:     "sse_kms_key_id",
 			Help:     "If using KMS ID you must provide the ARN of Key.",
-			Provider: "AWS,Ceph,ArvanCloud,Minio",
+			Provider: "AWS,Ceph,Minio",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "None",
@@ -1396,7 +1408,7 @@ isn't set then "acl" is used instead.`,
 		}, {
 			Name:     "sse_customer_key",
 			Help:     "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.",
-			Provider: "AWS,Ceph,ChinaMobile,ArvanCloud,Minio",
+			Provider: "AWS,Ceph,ChinaMobile,Minio",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -1408,7 +1420,7 @@ isn't set then "acl" is used instead.`,
 
 If you leave it blank, this is calculated automatically from the sse_customer_key provided.
 `,
-			Provider: "AWS,Ceph,ChinaMobile,ArvanCloud,Minio",
+			Provider: "AWS,Ceph,ChinaMobile,Minio",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -1488,9 +1500,6 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 			Help:     "The storage class to use when storing new objects in ArvanCloud.",
 			Provider: "ArvanCloud",
 			Examples: []fs.OptionExample{{
-				Value: "",
-				Help:  "Default",
-			}, {
 				Value: "STANDARD",
 				Help:  "Standard storage class",
 			}},
@@ -1826,6 +1835,20 @@ This should be true, false or left unset to use the default for the provider.
 `,
 			Default:  fs.Tristate{},
 			Advanced: true,
+		}, {
+			Name: "use_presigned_request",
+			Help: `Whether to use a presigned request or PutObject for single part uploads
+
+If this is false rclone will use PutObject from the AWS SDK to upload
+an object.
+
+Versions of rclone < 1.59 use presigned requests to upload a single
+part object and setting this flag to true will re-enable that
+functionality. This shouldn't be necessary except in exceptional
+circumstances or for testing.
+`,
+			Default:  false,
+			Advanced: true,
 		},
 		}})
 }
@@ -1891,6 +1914,7 @@ type Options struct {
 	DisableHTTP2          bool                 `config:"disable_http2"`
 	DownloadURL           string               `config:"download_url"`
 	UseMultipartEtag      fs.Tristate          `config:"use_multipart_etag"`
+	UsePresignedRequest   bool                 `config:"use_presigned_request"`
 }
 
 // Fs represents a remote s3 server
@@ -1902,6 +1926,7 @@ type Fs struct {
 	ctx           context.Context  // global context for reading config
 	features      *fs.Features     // optional features
 	c             *s3.S3           // the connection to the s3 server
+	cu            *s3.S3           // unsigned connection to the s3 server for PutObject
 	ses           *session.Session // the s3 session
 	rootBucket    string           // bucket part of root (if any)
 	rootDirectory string           // directory part of root (if any)
@@ -2034,7 +2059,11 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 }
 
 // s3Connection makes a connection to s3
-func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
+//
+// If unsignedBody is set then the connection is configured for
+// unsigned bodies which is necessary for PutObject if we don't want
+// it to seek
+func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *s3.S3, *session.Session, error) {
 	ci := fs.GetConfig(ctx)
 	// Make the auth
 	v := credentials.Value{
@@ -2051,7 +2080,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 	// start a new AWS session
 	awsSession, err := session.NewSession()
 	if err != nil {
-		return nil, nil, fmt.Errorf("NewSession: %w", err)
+		return nil, nil, nil, fmt.Errorf("NewSession: %w", err)
 	}
 
 	// first provider to supply a credential set "wins"
@@ -2091,9 +2120,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 		// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
 		cred = credentials.AnonymousCredentials
 	case v.AccessKeyID == "":
-		return nil, nil, errors.New("access_key_id not found")
+		return nil, nil, nil, errors.New("access_key_id not found")
 	case v.SecretAccessKey == "":
-		return nil, nil, errors.New("secret_access_key not found")
+		return nil, nil, nil, errors.New("secret_access_key not found")
 	}
 
 	if opt.Region == "" {
@@ -2132,25 +2161,36 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 		// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
 		awsSessionOpts.Config.Credentials = nil
 	}
+	// Setting this stops PutObject reading the body twice and seeking
+	// We add our own Content-MD5 for data protection
+	awsSessionOpts.Config.S3DisableContentMD5Validation = aws.Bool(true)
 	ses, err := session.NewSessionWithOptions(awsSessionOpts)
 	if err != nil {
-		return nil, nil, err
-	}
-	c := s3.New(ses)
-	if opt.V2Auth || opt.Region == "other-v2-signature" {
-		fs.Debugf(nil, "Using v2 auth")
-		signer := func(req *request.Request) {
-			// Ignore AnonymousCredentials object
-			if req.Config.Credentials == credentials.AnonymousCredentials {
-				return
+		return nil, nil, nil, err
+	}
+	newC := func(unsignedBody bool) *s3.S3 {
+		c := s3.New(ses)
+		if opt.V2Auth || opt.Region == "other-v2-signature" {
+			fs.Debugf(nil, "Using v2 auth")
+			signer := func(req *request.Request) {
+				// Ignore AnonymousCredentials object
+				if req.Config.Credentials == credentials.AnonymousCredentials {
+					return
+				}
+				sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
 			}
-			sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
+			c.Handlers.Sign.Clear()
+			c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+			c.Handlers.Sign.PushBack(signer)
+		} else if unsignedBody {
+			// If the body is unsigned then tell the signer that we aren't signing the payload
+			c.Handlers.Sign.Clear()
+			c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+			c.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler("v4.SignRequestHandler.WithUnsignedPayload", v4.WithUnsignedPayload))
 		}
-		c.Handlers.Sign.Clear()
-		c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
-		c.Handlers.Sign.PushBack(signer)
+		return c
 	}
-	return c, ses, nil
+	return newC(false), newC(true), ses, nil
 }
 
 func checkUploadChunkSize(cs fs.SizeSuffix) error {
@@ -2209,6 +2249,9 @@ func setQuirks(opt *Options) {
 		listObjectsV2 = false
 		virtualHostStyle = false
 		urlEncodeListings = false
+	case "Cloudflare":
+		virtualHostStyle = false
+		useMultipartEtag = false // currently multipart Etags are random
 	case "ArvanCloud":
 		listObjectsV2 = false
 		virtualHostStyle = false
@@ -2332,7 +2375,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
 	}
 	srv := getClient(ctx, opt)
-	c, ses, err := s3Connection(ctx, opt, srv)
+	c, cu, ses, err := s3Connection(ctx, opt, srv)
 	if err != nil {
 		return nil, err
 	}
@@ -2350,6 +2393,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ci:      ci,
 		ctx:     ctx,
 		c:       c,
+		cu:      cu,
 		ses:     ses,
 		pacer:   pc,
 		cache:   bucket.NewCache(),
@@ -2473,11 +2517,12 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
 	// Make a new session with the new region
 	oldRegion := f.opt.Region
 	f.opt.Region = region
-	c, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
+	c, cu, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
 	if err != nil {
 		return fmt.Errorf("creating new session failed: %w", err)
 	}
 	f.c = c
+	f.cu = cu
 	f.ses = ses
 
 	fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
@@ -3974,6 +4019,117 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	return etag, nil
 }
 
+// unWrapAwsError unwraps AWS errors, looking for a non AWS error
+//
+// It returns true if one was found and the error, or false and the
+// error passed in.
+func unWrapAwsError(err error) (found bool, outErr error) {
+	if awsErr, ok := err.(awserr.Error); ok {
+		var origErrs []error
+		if batchErr, ok := awsErr.(awserr.BatchError); ok {
+			origErrs = batchErr.OrigErrs()
+		} else {
+			origErrs = []error{awsErr.OrigErr()}
+		}
+		for _, origErr := range origErrs {
+			found, newErr := unWrapAwsError(origErr)
+			if found {
+				return found, newErr
+			}
+		}
+		return false, err
+	}
+	return true, err
+}
+
+// Upload a single part using PutObject
+func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
+	req.Body = readers.NewFakeSeeker(in, size)
+	var resp *s3.PutObjectOutput
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err = o.fs.cu.PutObject(req)
+		return o.fs.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		// Return the underlying error if we have a Serialization error if possible
+		//
+		// Serialization errors are synthesized locally in the SDK (not returned from the
+		// server). We'll get one if the SDK attempts a retry, however the FakeSeeker will
+		// remember the previous error from Read and return that.
+		if do, ok := err.(awserr.Error); ok && do.Code() == request.ErrCodeSerialization {
+			if found, newErr := unWrapAwsError(err); found {
+				err = newErr
+			}
+		}
+		return etag, lastModified, err
+	}
+	lastModified = time.Now()
+	etag = aws.StringValue(resp.ETag)
+	return etag, lastModified, nil
+}
+
+// Upload a single part using a presigned request
+func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
+	// Create the request
+	putObj, _ := o.fs.c.PutObjectRequest(req)
+
+	// Sign it so we can upload using a presigned request.
+	//
+	// Note the SDK didn't used to support streaming to
+	// PutObject so we used this work-around.
+	url, headers, err := putObj.PresignRequest(15 * time.Minute)
+	if err != nil {
+		return etag, lastModified, fmt.Errorf("s3 upload: sign request: %w", err)
+	}
+
+	if o.fs.opt.V2Auth && headers == nil {
+		headers = putObj.HTTPRequest.Header
+	}
+
+	// Set request to nil if empty so as not to make chunked encoding
+	if size == 0 {
+		in = nil
+	}
+
+	// create the vanilla http request
+	httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
+	if err != nil {
+		return etag, lastModified, fmt.Errorf("s3 upload: new request: %w", err)
+	}
+
+	// set the headers we signed and the length
+	httpReq.Header = headers
+	httpReq.ContentLength = size
+
+	var resp *http.Response
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		var err error
+		resp, err = o.fs.srv.Do(httpReq)
+		if err != nil {
+			return o.fs.shouldRetry(ctx, err)
+		}
+		body, err := rest.ReadBody(resp)
+		if err != nil {
+			return o.fs.shouldRetry(ctx, err)
+		}
+		if resp.StatusCode >= 200 && resp.StatusCode < 299 {
+			return false, nil
+		}
+		err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
+		return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	})
+	if err != nil {
+		return etag, lastModified, err
+	}
+	if resp != nil {
+		if date, err := http.ParseTime(resp.Header.Get("Date")); err != nil {
+			lastModified = date
+		}
+		etag = resp.Header.Get("Etag")
+	}
+	return etag, lastModified, nil
+}
+
 // Update the Object from in with modTime and size
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	bucket, bucketPath := o.split()
@@ -4025,6 +4181,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		ContentType: &mimeType,
 		Metadata:    metadata,
 	}
+	if size >= 0 {
+		req.ContentLength = &size
+	}
 	if md5sumBase64 != "" {
 		req.ContentMD5 = &md5sumBase64
 	}
@@ -4079,66 +4238,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
-	var resp *http.Response // response from PUT
-	var wantETag string     // Multipart upload Etag to check
+	var wantETag string        // Multipart upload Etag to check
+	var gotEtag string         // Etag we got from the upload
+	var lastModified time.Time // Time we got from the upload
 	if multipart {
 		wantETag, err = o.uploadMultipart(ctx, &req, size, in)
-		if err != nil {
-			return err
-		}
 	} else {
-
-		// Create the request
-		putObj, _ := o.fs.c.PutObjectRequest(&req)
-
-		// Sign it so we can upload using a presigned request.
-		//
-		// Note the SDK doesn't currently support streaming to
-		// PutObject so we'll use this work-around.
-		url, headers, err := putObj.PresignRequest(15 * time.Minute)
-		if err != nil {
-			return fmt.Errorf("s3 upload: sign request: %w", err)
-		}
-
-		if o.fs.opt.V2Auth && headers == nil {
-			headers = putObj.HTTPRequest.Header
-		}
-
-		// Set request to nil if empty so as not to make chunked encoding
-		if size == 0 {
-			in = nil
-		}
-
-		// create the vanilla http request
-		httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
-		if err != nil {
-			return fmt.Errorf("s3 upload: new request: %w", err)
-		}
-
-		// set the headers we signed and the length
-		httpReq.Header = headers
-		httpReq.ContentLength = size
-
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			var err error
-			resp, err = o.fs.srv.Do(httpReq)
-			if err != nil {
-				return o.fs.shouldRetry(ctx, err)
-			}
-			body, err := rest.ReadBody(resp)
-			if err != nil {
-				return o.fs.shouldRetry(ctx, err)
-			}
-			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
-				return false, nil
-			}
-			err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
-			return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-		})
-		if err != nil {
-			return err
+		if o.fs.opt.UsePresignedRequest {
+			gotEtag, lastModified, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
+		} else {
+			gotEtag, lastModified, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
 		}
 	}
+	if err != nil {
+		return err
+	}
 
 	// User requested we don't HEAD the object after uploading it
 	// so make up the object as best we can assuming it got
@@ -4151,11 +4265,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		o.mimeType = aws.StringValue(req.ContentType)
 		o.storageClass = aws.StringValue(req.StorageClass)
 		// If we have done a single part PUT request then we can read these
-		if resp != nil {
-			if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
-				o.lastModified = date
-			}
-			o.setMD5FromEtag(resp.Header.Get("Etag"))
+		if gotEtag != "" {
+			o.setMD5FromEtag(gotEtag)
+		}
+		if !o.lastModified.IsZero() {
+			o.lastModified = lastModified
 		}
 		return nil
 	}
diff --git a/docs/content/_index.md b/docs/content/_index.md
index 1f595763d..167c0749d 100644
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@@ -116,6 +116,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
 {{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
 {{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/s3/#scaleway" >}}
+{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
 {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 {{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 779541631..2ebd2d192 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -588,3 +588,5 @@ put them back in again.` >}}
   * Lesmiscore <nao20010128@gmail.com>
   * ehsantdy <ehsan.tadayon@arvancloud.com>
   * SwazRGB <65694696+swazrgb@users.noreply.github.com>
+  * Mateusz Puczyński <mati6095@gmail.com>
+  * Michael C Tiernan - MIT-Research Computing Project <mtiernan@mit.edu>
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index 7107e3253..8ab466527 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -700,7 +700,7 @@ consider using the flag
 
 Google docs exist as virtual files on Google Drive and cannot be transferred
 to other filesystems natively. While it is possible to export a Google doc to
-a normal file (with `.xlsx` extension, for example), it's not possible
+a normal file (with `.xlsx` extension, for example), it is not possible
 to import a normal file back into a Google document.
 
 Bisync's handling of Google Doc files is to flag them in the run log output
diff --git a/docs/content/faq.md b/docs/content/faq.md
index f6f31a1b6..ecd240882 100644
--- a/docs/content/faq.md
+++ b/docs/content/faq.md
@@ -31,7 +31,7 @@ the node running rclone would need to have lots of bandwidth.
 
 The syncs would be incremental (on a file by file basis).
 
-Eg
+e.g.
 
     rclone sync -i drive:Folder s3:bucket
 
diff --git a/docs/content/install.md b/docs/content/install.md
index 52e63f534..eccea9543 100644
--- a/docs/content/install.md
+++ b/docs/content/install.md
@@ -26,11 +26,11 @@ using the [rclone selfupdate](/commands/rclone_selfupdate/) command.
 
 To install rclone on Linux/macOS/BSD systems, run:
 
-    curl https://rclone.org/install.sh | sudo bash
+    sudo -v ; curl https://rclone.org/install.sh | sudo bash
 
 For beta installation, run:
 
-    curl https://rclone.org/install.sh | sudo bash -s beta
+    sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
 
 Note that this script checks the version of rclone installed first and
 won't re-download if not needed.
diff --git a/docs/content/install.sh b/docs/content/install.sh
index 3294c1563..8a8242fee 100755
--- a/docs/content/install.sh
+++ b/docs/content/install.sh
@@ -12,7 +12,7 @@ set -e
 #when adding a tool to the list make sure to also add its corresponding command further in the script
 unzip_tools_list=('unzip' '7z' 'busybox')
 
-usage() { echo "Usage: curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
+usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
 
 #check for beta flag
 if [ -n "$1" ] && [ "$1" != "beta" ]; then
@@ -81,6 +81,8 @@ case $OS in
     ;;  
   Darwin)
     OS='osx'
+    binTgtDir=/usr/local/bin
+    man1TgtDir=/usr/local/share/man/man1
     ;;
   SunOS)
     OS='solaris'
@@ -171,12 +173,14 @@ case "$OS" in
     ;;
   'osx')
     #binary
-    mkdir -p /usr/local/bin
-    cp rclone /usr/local/bin/rclone.new
-    mv /usr/local/bin/rclone.new /usr/local/bin/rclone
+    mkdir -m 0555 -p ${binTgtDir}
+    cp rclone ${binTgtDir}/rclone.new
+    mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone
+    chmod a=x ${binTgtDir}/rclone
     #manual
-    mkdir -p /usr/local/share/man/man1
-    cp rclone.1 /usr/local/share/man/man1/    
+    mkdir -m 0555 -p ${man1TgtDir}
+    cp rclone.1 ${man1TgtDir}    
+    chmod a=r ${man1TgtDir}/rclone.1
     ;;
   *)
     echo 'OS not supported'
diff --git a/docs/content/s3.md b/docs/content/s3.md
index 0cc46fe25..aaf76804d 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -12,6 +12,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
 {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
 {{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
+{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
 {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
 {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@@ -1728,7 +1729,7 @@ The storage class to use when storing new objects in ChinaMobile.
 
 #### --s3-storage-class
 
-The storage class to use when storing new objects in ChinaMobile.
+The storage class to use when storing new objects in ArvanCloud.
 
 - Config:      storage_class
 - Env Var:     RCLONE_S3_STORAGE_CLASS
@@ -1736,8 +1737,6 @@ The storage class to use when storing new objects in ChinaMobile.
 - Type:        string
 - Required:    false
 - Examples:
-    - ""
-        - Default
     - "STANDARD"
         - Standard storage class
 
@@ -2481,12 +2480,19 @@ You will be able to list and copy data but not upload it.
 This is the provider used as main example and described in the [configuration](#configuration) section above.
 
 ### AWS Snowball Edge
-[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware appliance used for transferring
-bulk data back to AWS. Its main software interface is S3 object storage.
 
-To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service'
-be sure to set `upload_cutoff = 0` otherwise you will run into authentication header issues as
-the snowball device does not support query parameter based authentication.
+[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware
+appliance used for transferring bulk data back to AWS. Its main
+software interface is S3 object storage.
+
+To use rclone with AWS Snowball Edge devices, configure as standard
+for an 'S3 Compatible Service'.
+
+If using rclone pre v1.59 be sure to set `upload_cutoff = 0` otherwise
+you will run into authentication header issues as the snowball device
+does not support query parameter based authentication.
+
+With rclone v1.59 or later setting `upload_cutoff` should not be necessary.
 
 eg.
 ```
@@ -2525,10 +2531,11 @@ server_side_encryption =
 storage_class =
 ```
 
-If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you
-may need to supply the parameter `--s3-upload-cutoff 0` or put this in
-the config file as `upload_cutoff 0` to work around a bug which causes
-uploading of small files to fail.
+If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a
+version of rclone before v1.59 then you may need to supply the
+parameter `--s3-upload-cutoff 0` or put this in the config file as
+`upload_cutoff 0` to work around a bug which causes uploading of small
+files to fail.
 
 Note also that Ceph sometimes puts `/` in the passwords it gives
 users.  If you read the secret access key using the command line tools
@@ -2555,6 +2562,115 @@ removed).
 Because this is a json dump, it is encoding the `/` as `\/`, so if you
 use the secret key as `xxxxxx/xxxx`  it will work fine.
 
+### Cloudflare R2 {#cloudflare-r2}
+
+[Cloudflare R2](https://blog.cloudflare.com/r2-open-beta/) Storage
+allows developers to store large amounts of unstructured data without
+the costly egress bandwidth fees associated with typical cloud storage
+services.
+
+Here is an example of making a Cloudflare R2 configuration. First run:
+
+    rclone config
+
+This will guide you through an interactive setup process.
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> r2
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+...
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+...
+Storage> s3
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+...
+XX / Cloudflare R2 Storage
+   \ (Cloudflare)
+...
+provider> Cloudflare
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth> 1
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> ACCESS_KEY
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> SECRET_ACCESS_KEY
+Option region.
+Region to connect to.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
+   \ (auto)
+region> 1
+Option endpoint.
+Endpoint for S3 API.
+Required when using an S3 clone.
+Enter a value. Press Enter to leave empty.
+endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+...
+acl> 1
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+This will leave your config looking something like:
+
+```
+[r2]
+type = s3
+provider = Cloudflare
+access_key_id = ACCESS_KEY
+secret_access_key = SECRET_ACCESS_KEY
+region = auto
+endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com
+acl = private
+```
+
+Now run `rclone lsf r2:` to see your buckets and `rclone lsf
+r2:bucket` to look within a bucket.
+
 ### Dreamhost
 
 Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is
diff --git a/fs/mount_helper.go b/fs/mount_helper.go
index 884b1f67c..f45b85895 100644
--- a/fs/mount_helper.go
+++ b/fs/mount_helper.go
@@ -208,7 +208,7 @@ func parseHelperOptionString(optString string) (opts []string, err error) {
 				if len(param) == 0 {
 					return nil, errHelperEmptyOption
 				}
-				if param[0] == '-' || param[0] == '_' {
+				if param[0] == '-' {
 					return nil, errHelperOptionName
 				}
 				prev = i + 1
diff --git a/fs/open_options.go b/fs/open_options.go
index 8d997118a..84e43045c 100644
--- a/fs/open_options.go
+++ b/fs/open_options.go
@@ -138,8 +138,14 @@ func (o *RangeOption) Decode(size int64) (offset, limit int64) {
 // absolute fetch using the size passed in and makes sure the range does
 // not exceed filesize. Some remotes (e.g. Onedrive, Box) don't support
 // range requests which index from the end.
+//
+// It also adjusts any SeekOption~s, turning them into absolute
+// RangeOption~s instead.
 func FixRangeOption(options []OpenOption, size int64) {
-	if size == 0 {
+	if size < 0 {
+		// Can't do anything for unknown length objects
+		return
+	} else if size == 0 {
 		// if size 0 then remove RangeOption~s
 		// replacing with a NullOptions~s which won't be rendered
 		for i := range options {
@@ -150,18 +156,21 @@ func FixRangeOption(options []OpenOption, size int64) {
 		}
 		return
 	}
-	for i := range options {
-		option := options[i]
-		if x, ok := option.(*RangeOption); ok {
+	for i, option := range options {
+		switch x := option.(type) {
+		case *RangeOption:
 			// If start is < 0 then fetch from the end
 			if x.Start < 0 {
 				x = &RangeOption{Start: size - x.End, End: -1}
 				options[i] = x
 			}
-			if x.End > size {
+			// If end is too big or undefined, fetch to the end
+			if x.End > size || x.End < 0 {
 				x = &RangeOption{Start: x.Start, End: size - 1}
 				options[i] = x
 			}
+		case *SeekOption:
+			options[i] = &RangeOption{Start: x.Offset, End: size - 1}
 		}
 	}
 }
diff --git a/lib/readers/fakeseeker.go b/lib/readers/fakeseeker.go
new file mode 100644
index 000000000..844fc0422
--- /dev/null
+++ b/lib/readers/fakeseeker.go
@@ -0,0 +1,72 @@
+package readers
+
+import (
+	"errors"
+	"fmt"
+	"io"
+)
+
+// FakeSeeker adapts an io.Seeker into an io.ReadSeeker
+type FakeSeeker struct {
+	in      io.Reader
+	readErr error
+	length  int64
+	offset  int64
+	read    bool
+}
+
+// NewFakeSeeker creates a fake io.ReadSeeker from an io.Reader
+//
+// This can be seeked before reading to discover the length passed in.
+func NewFakeSeeker(in io.Reader, length int64) io.ReadSeeker {
+	if rs, ok := in.(io.ReadSeeker); ok {
+		return rs
+	}
+	return &FakeSeeker{
+		in:     in,
+		length: length,
+	}
+}
+
+// Seek the stream - possible only before reading
+func (r *FakeSeeker) Seek(offset int64, whence int) (abs int64, err error) {
+	if r.readErr != nil {
+		return 0, r.readErr
+	}
+	if r.read {
+		return 0, fmt.Errorf("FakeSeeker: can't Seek(%d, %d) after reading", offset, whence)
+	}
+	switch whence {
+	case io.SeekStart:
+		abs = offset
+	case io.SeekCurrent:
+		abs = r.offset + offset
+	case io.SeekEnd:
+		abs = r.length + offset
+	default:
+		return 0, errors.New("FakeSeeker: invalid whence")
+	}
+	if abs < 0 {
+		return 0, errors.New("FakeSeeker: negative position")
+	}
+	r.offset = abs
+	return abs, nil
+}
+
+// Read data from the stream. Will give an error if seeked.
+func (r *FakeSeeker) Read(p []byte) (n int, err error) {
+	if r.readErr != nil {
+		return 0, r.readErr
+	}
+	if !r.read && r.offset != 0 {
+		return 0, errors.New("FakeSeeker: not at start: can't read")
+	}
+	n, err = r.in.Read(p)
+	if n != 0 {
+		r.read = true
+	}
+	if err != nil {
+		r.readErr = err
+	}
+	return n, err
+}


Test patch:
diff --git a/fs/mount_helper_test.go b/fs/mount_helper_test.go
index 00c4fba79..3235b1195 100644
--- a/fs/mount_helper_test.go
+++ b/fs/mount_helper_test.go
@@ -20,7 +20,7 @@ func TestMountHelperArgs(t *testing.T) {
 		src: []string{},
 		dst: []string{"mount", "--daemon"},
 	}, {
-		src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env`},
+		src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env,_netdev`},
 		dst: []string{"mount", "--read-only", "--verbose=3", "--daemon"},
 		env: "HTTPS_PROXY=a b;c,d?EF",
 	}}
diff --git a/fs/open_options_test.go b/fs/open_options_test.go
index 52fc42ddb..829f45c19 100644
--- a/fs/open_options_test.go
+++ b/fs/open_options_test.go
@@ -149,6 +149,16 @@ func TestFixRangeOptions(t *testing.T) {
 			in:   []OpenOption{},
 			want: []OpenOption{},
 		},
+		{
+			name: "Unknown size -1",
+			in: []OpenOption{
+				&RangeOption{Start: 1, End: -1},
+			},
+			want: []OpenOption{
+				&RangeOption{Start: 1, End: -1},
+			},
+			size: -1,
+		},
 		{
 			name: "Fetch a range with size=0",
 			in: []OpenOption{
@@ -183,7 +193,7 @@ func TestFixRangeOptions(t *testing.T) {
 				&RangeOption{Start: 1, End: -1},
 			},
 			want: []OpenOption{
-				&RangeOption{Start: 1, End: -1},
+				&RangeOption{Start: 1, End: 99},
 			},
 			size: 100,
 		},
@@ -193,7 +203,7 @@ func TestFixRangeOptions(t *testing.T) {
 				&RangeOption{Start: -1, End: 10},
 			},
 			want: []OpenOption{
-				&RangeOption{Start: 90, End: -1},
+				&RangeOption{Start: 90, End: 99},
 			},
 			size: 100,
 		},
@@ -207,6 +217,20 @@ func TestFixRangeOptions(t *testing.T) {
 			},
 			size: 100,
 		},
+		{
+			name: "SeekOption",
+			in: []OpenOption{
+				&HTTPOption{Key: "a", Value: "1"},
+				&SeekOption{Offset: 10},
+				&HTTPOption{Key: "b", Value: "2"},
+			},
+			want: []OpenOption{
+				&HTTPOption{Key: "a", Value: "1"},
+				&RangeOption{Start: 10, End: 99},
+				&HTTPOption{Key: "b", Value: "2"},
+			},
+			size: 100,
+		},
 	} {
 		FixRangeOption(test.in, test.size)
 		assert.Equal(t, test.want, test.in, test.name)
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index ca36aa7d8..5b4dd86e6 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -204,6 +204,11 @@ backends:
  - backend:  "s3"
    remote:   "TestS3Alibaba:"
    fastlist: true
+ - backend:  "s3"
+   remote:   "TestS3R2:"
+   fastlist: true
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/SetTier
  - backend:  "sftp"
    remote:   "TestSFTPOpenssh:"
    fastlist: false
diff --git a/lib/readers/fakeseeker_test.go b/lib/readers/fakeseeker_test.go
new file mode 100644
index 000000000..0702b7cbb
--- /dev/null
+++ b/lib/readers/fakeseeker_test.go
@@ -0,0 +1,87 @@
+package readers
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Check interface
+var _ io.ReadSeeker = &FakeSeeker{}
+
+func TestFakeSeeker(t *testing.T) {
+	// Test that passing in an io.ReadSeeker just passes it through
+	bufReader := bytes.NewReader([]byte{1})
+	r := NewFakeSeeker(bufReader, 5)
+	assert.Equal(t, r, bufReader)
+
+	in := bytes.NewBufferString("hello")
+	buf := make([]byte, 16)
+	r = NewFakeSeeker(in, 5)
+	assert.NotEqual(t, r, in)
+
+	// check the seek offset is as passed in
+	checkPos := func(pos int64) {
+		abs, err := r.Seek(0, io.SeekCurrent)
+		require.NoError(t, err)
+		assert.Equal(t, pos, abs)
+	}
+
+	// Test some seeking
+	checkPos(0)
+
+	abs, err := r.Seek(2, io.SeekStart)
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), abs)
+	checkPos(2)
+
+	abs, err = r.Seek(-1, io.SeekEnd)
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), abs)
+	checkPos(4)
+
+	// Check can't read if not at start
+	_, err = r.Read(buf)
+	require.ErrorContains(t, err, "not at start")
+
+	// Seek back to start
+	abs, err = r.Seek(-4, io.SeekCurrent)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), abs)
+	checkPos(0)
+
+	_, err = r.Seek(42, 17)
+	require.ErrorContains(t, err, "invalid whence")
+
+	_, err = r.Seek(-1, io.SeekStart)
+	require.ErrorContains(t, err, "negative position")
+
+	// Test reading now seeked back to the start
+	n, err := r.Read(buf)
+	require.NoError(t, err)
+	assert.Equal(t, 5, n)
+	assert.Equal(t, []byte("hello"), buf[:5])
+
+	// Seeking should give an error now
+	_, err = r.Seek(-1, io.SeekEnd)
+	require.ErrorContains(t, err, "after reading")
+}
+
+func TestFakeSeekerError(t *testing.T) {
+	in := bytes.NewBufferString("hello")
+	r := NewFakeSeeker(in, 5)
+	assert.NotEqual(t, r, in)
+
+	buf, err := io.ReadAll(r)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("hello"), buf)
+
+	_, err = r.Read(buf)
+	assert.Equal(t, io.EOF, err)
+
+	_, err = r.Seek(0, io.SeekStart)
+	assert.Equal(t, io.EOF, err)
+}