Repository: rclone/rclone
Pull request #6138 (2022-05-01 21:34:52)

B2: Dynamically adjust --b2-chunk-size when necessary
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?
<!--
Describe the changes here
-->
Add support for dynamic modification of `--b2-chunk-size` when necessary as discussed in #4643.

Created a new package, `chunksize` that implements a chunk size calculator, which is then called by B2 to dynamically adjust the chunk size when needed.  This package was created with the intent to eventually leverage this calculator in all backends that need to do dynamic chunk size adjustments (S3 at a minimum).

#### Was the change discussed in an issue or in the forum before?

<!--
Link issues and relevant forum posts here.
-->
Yes.  #4643
#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)


Slugger: Thanks for the feedback. I've addressed all the comments except the implementation in additional backends.  I'll do that next time I can put some time in -- some evening hopefully later this week.

Just dipping my toes into the project here with this relatively simple feature request I found in the backlog. Got to say, so far so good.  It's always nice to get a super quick, constructive response when starting out in a new project. Looking forward to contributing (I have a big idea, which is what brought me here, but learning to crawl before I try to climb Everest! 😄 ).
Slugger: Refactored `s3` and `azureblob` to leverage the new `chunksize.Calculator()`
Slugger: Linter issues fixed, commits squashed.  Everything should be ready to go.

Patch:
diff --git a/README.md b/README.md
index 1f3ca3dfb..0235323b5 100644
--- a/README.md
+++ b/README.md
@@ -29,6 +29,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
   * China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
+  * Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
   * Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 880d3eefb..f03ff72ef 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -26,6 +26,7 @@ import (
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
@@ -1690,25 +1691,17 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
-	// calculate size of parts/blocks
-	partSize := int(o.fs.opt.ChunkSize)
-
 	uploadParts := int64(maxUploadParts)
 	if uploadParts < 1 {
 		uploadParts = 1
 	} else if uploadParts > maxUploadParts {
 		uploadParts = maxUploadParts
 	}
-
-	// Adjust partSize until the number of parts/blocks is small enough.
-	if o.size/int64(partSize) >= uploadParts {
-		// Calculate partition size rounded up to the nearest MiB
-		partSize = int((((o.size / uploadParts) >> 20) + 1) << 20)
-		fs.Debugf(o, "Adjust partSize to %q", partSize)
-	}
+	// calculate size of parts/blocks
+	partSize := chunksize.Calculator(o, int(uploadParts), o.fs.opt.ChunkSize)
 
 	putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
-		BufferSize:      partSize,
+		BufferSize:      int(partSize),
 		MaxBuffers:      o.fs.opt.UploadConcurrency,
 		Metadata:        o.meta,
 		BlobHTTPHeaders: httpHeaders,
diff --git a/backend/b2/upload.go b/backend/b2/upload.go
index 827d33578..8a1f67635 100644
--- a/backend/b2/upload.go
+++ b/backend/b2/upload.go
@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/rest"
@@ -88,21 +89,19 @@ type largeUpload struct {
 // newLargeUpload starts an upload of object o from in with metadata in src
 //
 // If newInfo is set then metadata from that will be used instead of reading it from src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
-	remote := o.remote
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	size := src.Size()
 	parts := int64(0)
 	sha1SliceSize := int64(maxParts)
+	chunkSize := defaultChunkSize
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
+		chunkSize = chunksize.Calculator(src, maxParts, defaultChunkSize)
 		parts = size / int64(chunkSize)
 		if size%int64(chunkSize) != 0 {
 			parts++
 		}
-		if parts > maxParts {
-			return nil, fmt.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts)
-		}
 		sha1SliceSize = parts
 	}
 
diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go
index ce994c23c..ce021e35f 100644
--- a/backend/internetarchive/internetarchive.go
+++ b/backend/internetarchive/internetarchive.go
@@ -28,6 +28,7 @@ import (
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -129,6 +130,7 @@ type IAFile struct {
 	// Source     string `json:"source"`
 	Mtime       string          `json:"mtime"`
 	RcloneMtime json.RawMessage `json:"rclone-mtime"`
+	UpdateTrack json.RawMessage `json:"rclone-update-track"`
 	Size        string          `json:"size"`
 	Md5         string          `json:"md5"`
 	Crc32       string          `json:"crc32"`
@@ -294,7 +296,7 @@ func (o *Object) Storable() bool {
 	return true
 }
 
-// SetModTime is not supported
+// SetModTime sets modTime on a particular file
 func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	bucket, reqDir := o.split()
 	if bucket == "" {
@@ -483,6 +485,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		return nil, fs.ErrorCantCopy
 	}
 
+	updateTracker := random.String(32)
 	headers := map[string]string{
 		"x-archive-auto-make-bucket": "1",
 		"x-archive-queue-derive":     "0",
@@ -494,7 +497,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		"x-archive-filemeta-crc32":   srcObj.crc32,
 		"x-archive-filemeta-size":    fmt.Sprint(srcObj.size),
 		// add this too for sure
-		"x-archive-filemeta-rclone-mtime": srcObj.modTime.Format(time.RFC3339Nano),
+		"x-archive-filemeta-rclone-mtime":        srcObj.modTime.Format(time.RFC3339Nano),
+		"x-archive-filemeta-rclone-update-track": updateTracker,
 	}
 
 	// make a PUT request at (IAS3)/:item/:path without body
@@ -515,7 +519,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 
 	// we can't update/find metadata here as IA will also
 	// queue server-side copy as well as upload/delete.
-	return f.waitFileUpload(ctx, trimPathPrefix(path.Join(dstBucket, dstPath), f.root, f.opt.Enc), f.getHashes(ctx, src), srcObj.size)
+	return f.waitFileUpload(ctx, trimPathPrefix(path.Join(dstBucket, dstPath), f.root, f.opt.Enc), updateTracker, srcObj.size)
 }
 
 // ListR lists the objects and directories of the Fs starting
@@ -660,12 +664,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	bucket, bucketPath := o.split()
 	modTime := src.ModTime(ctx)
 	size := src.Size()
+	updateTracker := random.String(32)
 
 	// Set the mtime in the metadata
 	// internetarchive backend builds at header level as IAS3 has extension outside X-Amz-
 	headers := map[string]string{
 		// https://github.com/jjjake/internetarchive/blob/2456376533251df9d05e0a14d796ec1ced4959f5/internetarchive/iarequest.py#L158
-		"x-amz-filemeta-rclone-mtime": modTime.Format(time.RFC3339Nano),
+		"x-amz-filemeta-rclone-mtime":        modTime.Format(time.RFC3339Nano),
+		"x-amz-filemeta-rclone-update-track": updateTracker,
 
 		// we add some more headers for intuitive actions
 		"x-amz-auto-make-bucket":     "1",    // create an item if does not exist, do nothing if already
@@ -712,7 +718,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)
 	var newObj *Object
 	if err == nil {
-		newObj, err = o.fs.waitFileUpload(ctx, o.remote, o.fs.getHashes(ctx, src), size)
+		newObj, err = o.fs.waitFileUpload(ctx, o.remote, updateTracker, size)
 	} else {
 		newObj = &Object{}
 	}
@@ -782,18 +788,6 @@ func (o *Object) split() (bucket, bucketPath string) {
 	return o.fs.split(o.remote)
 }
 
-func (f *Fs) getHashes(ctx context.Context, src fs.ObjectInfo) map[hash.Type]string {
-	hashMap := map[hash.Type]string{}
-	for _, ty := range f.Hashes().Array() {
-		sum, err := src.Hash(ctx, ty)
-		if err != nil || sum == "" {
-			continue
-		}
-		hashMap[ty] = sum
-	}
-	return hashMap
-}
-
 func (f *Fs) requestMetadata(ctx context.Context, bucket string) (result MetadataResponse, err error) {
 	var resp *http.Response
 	// make a GET request to (frontend)/metadata/:item/
@@ -852,7 +846,7 @@ func (f *Fs) listAllUnconstrained(ctx context.Context, bucket string) (entries f
 	return entries, nil
 }
 
-func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[hash.Type]string, newSize int64) (ret *Object, err error) {
+func (f *Fs) waitFileUpload(ctx context.Context, reqPath, tracker string, newSize int64) (ret *Object, err error) {
 	bucket, bucketPath := f.split(reqPath)
 
 	ret = &Object{
@@ -869,6 +863,10 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 			ret2, ok := ret2.(*Object)
 			if ok {
 				ret = ret2
+				ret.crc32 = ""
+				ret.md5 = ""
+				ret.sha1 = ""
+				ret.size = -1
 			}
 		}
 		return ret, nil
@@ -881,9 +879,6 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 	go func() {
 		isFirstTime := true
 		existed := false
-		oldMtime := ""
-		oldCrc32 := ""
-		unreliablePassCount := 0
 		for {
 			if !isFirstTime {
 				// depending on the queue, it takes time
@@ -908,10 +903,6 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 			if isFirstTime {
 				isFirstTime = false
 				existed = iaFile != nil
-				if iaFile != nil {
-					oldMtime = iaFile.Mtime
-					oldCrc32 = iaFile.Crc32
-				}
 			}
 			if iaFile == nil {
 				continue
@@ -925,38 +916,20 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath string, newHashes map[h
 				return
 			}
 
-			hashMatched := true
-			for tt, sum := range newHashes {
-				if tt == hash.MD5 && !hash.Equals(iaFile.Md5, sum) {
-					hashMatched = false
-					break
-				}
-				if tt == hash.SHA1 && !hash.Equals(iaFile.Sha1, sum) {
-					hashMatched = false
-					break
-				}
-				if tt == hash.CRC32 && !hash.Equals(iaFile.Crc32, sum) {
-					hashMatched = false
+			fileTrackers, _ := listOrString(iaFile.UpdateTrack)
+			trackerMatch := false
+			for _, v := range fileTrackers {
+				if v == tracker {
+					trackerMatch = true
 					break
 				}
 			}
-			if !hashMatched {
+			if !trackerMatch {
 				continue
 			}
 			if !compareSize(parseSize(iaFile.Size), newSize) {
 				continue
 			}
-			if hash.Equals(oldCrc32, iaFile.Crc32) && unreliablePassCount < 60 {
-				// the following two are based on a sort of "bad" assumption;
-				// what if the file is updated immediately, before polling?
-				// by limiting hits of these tests, avoid infinite loop
-				unreliablePassCount++
-				continue
-			}
-			if hash.Equals(iaFile.Mtime, oldMtime) && unreliablePassCount < 60 {
-				unreliablePassCount++
-				continue
-			}
 
 			// voila!
 			retC <- struct {
@@ -1036,20 +1009,24 @@ func makeValidObject2(f *Fs, file IAFile, bucket string) *Object {
 	return makeValidObject(f, trimPathPrefix(path.Join(bucket, file.Name), f.root, f.opt.Enc), file, mtimeTime, size)
 }
 
-func (file IAFile) parseMtime() (mtime time.Time) {
-	// method 1: use metadata added by rclone
-	var rmArray []string
+func listOrString(jm json.RawMessage) (rmArray []string, err error) {
 	// rclone-metadata can be an array or string
 	// try to deserialize it as array first
-	err := json.Unmarshal(file.RcloneMtime, &rmArray)
+	err = json.Unmarshal(jm, &rmArray)
 	if err != nil {
 		// if not, it's a string
 		dst := new(string)
-		err = json.Unmarshal(file.RcloneMtime, dst)
+		err = json.Unmarshal(jm, dst)
 		if err == nil {
 			rmArray = []string{*dst}
 		}
 	}
+	return
+}
+
+func (file IAFile) parseMtime() (mtime time.Time) {
+	// method 1: use metadata added by rclone
+	rmArray, err := listOrString(file.RcloneMtime)
 	// let's take the first value we can deserialize
 	for _, value := range rmArray {
 		mtime, err = time.Parse(time.RFC3339Nano, value)
diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go
index 62491dea1..5d3838254 100644
--- a/backend/jottacloud/jottacloud.go
+++ b/backend/jottacloud/jottacloud.go
@@ -932,7 +932,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	return entries, nil
 }
 
-func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, filesystem *Fs, callback func(fs.DirEntry) error) error {
+func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {
 
 	type stats struct {
 		Folders int `xml:"folders"`
@@ -968,8 +968,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
 		})
 	}
 
+	// liststream paths are /mountpoint/root/path
+	// so the returned paths should have /mountpoint/root/ trimmed
+	// as the caller is expecting path.
+	pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
 	trimPathPrefix := func(p string) string {
-		p = strings.TrimPrefix(p, trimPrefix)
+		p = strings.TrimPrefix(p, pathPrefix)
 		p = strings.TrimPrefix(p, "/")
 		return p
 	}
@@ -1052,11 +1056,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 			return shouldRetry(ctx, resp, err)
 		}
 
-		// liststream paths are /mountpoint/root/path
-		// so the returned paths should have /mountpoint/root/ trimmed
-		// as the caller is expecting path.
-		trimPrefix := path.Join("/", f.opt.Mountpoint, f.root)
-		err = parseListRStream(ctx, resp.Body, trimPrefix, f, func(d fs.DirEntry) error {
+		err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
 			if d.Remote() == dir {
 				return nil
 			}
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 81f071b92..434fd7752 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -32,9 +32,11 @@ import (
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/ncw/swift/v2"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
@@ -58,7 +60,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Options: []fs.Option{{
@@ -78,6 +80,9 @@ func init() {
 			}, {
 				Value: "ChinaMobile",
 				Help:  "China Mobile Ecloud Elastic Object Storage (EOS)",
+			}, {
+				Value: "Cloudflare",
+				Help:  "Cloudflare R2 Storage",
 			}, {
 				Value: "ArvanCloud",
 				Help:  "Arvan Cloud Object Storage (AOS)",
@@ -297,10 +302,18 @@ func init() {
 				Value: "fr-par",
 				Help:  "Paris, France",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region to connect to.",
+			Provider: "Cloudflare",
+			Examples: []fs.OptionExample{{
+				Value: "auto",
+				Help:  "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
+			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -1270,7 +1283,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
+			Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1823,6 +1836,20 @@ This should be true, false or left unset to use the default for the provider.
 `,
 			Default:  fs.Tristate{},
 			Advanced: true,
+		}, {
+			Name: "use_presigned_request",
+			Help: `Whether to use a presigned request or PutObject for single part uploads
+
+If this is false rclone will use PutObject from the AWS SDK to upload
+an object.
+
+Versions of rclone < 1.59 use presigned requests to upload a single
+part object and setting this flag to true will re-enable that
+functionality. This shouldn't be necessary except in exceptional
+circumstances or for testing.
+`,
+			Default:  false,
+			Advanced: true,
 		},
 		}})
 }
@@ -1888,6 +1915,7 @@ type Options struct {
 	DisableHTTP2          bool                 `config:"disable_http2"`
 	DownloadURL           string               `config:"download_url"`
 	UseMultipartEtag      fs.Tristate          `config:"use_multipart_etag"`
+	UsePresignedRequest   bool                 `config:"use_presigned_request"`
 }
 
 // Fs represents a remote s3 server
@@ -1899,6 +1927,7 @@ type Fs struct {
 	ctx           context.Context  // global context for reading config
 	features      *fs.Features     // optional features
 	c             *s3.S3           // the connection to the s3 server
+	cu            *s3.S3           // unsigned connection to the s3 server for PutObject
 	ses           *session.Session // the s3 session
 	rootBucket    string           // bucket part of root (if any)
 	rootDirectory string           // directory part of root (if any)
@@ -2031,7 +2060,11 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 }
 
 // s3Connection makes a connection to s3
-func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
+//
+// If unsignedBody is set then the connection is configured for
+// unsigned bodies which is necessary for PutObject if we don't want
+// it to seek
+func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *s3.S3, *session.Session, error) {
 	ci := fs.GetConfig(ctx)
 	// Make the auth
 	v := credentials.Value{
@@ -2048,7 +2081,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 	// start a new AWS session
 	awsSession, err := session.NewSession()
 	if err != nil {
-		return nil, nil, fmt.Errorf("NewSession: %w", err)
+		return nil, nil, nil, fmt.Errorf("NewSession: %w", err)
 	}
 
 	// first provider to supply a credential set "wins"
@@ -2088,9 +2121,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 		// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
 		cred = credentials.AnonymousCredentials
 	case v.AccessKeyID == "":
-		return nil, nil, errors.New("access_key_id not found")
+		return nil, nil, nil, errors.New("access_key_id not found")
 	case v.SecretAccessKey == "":
-		return nil, nil, errors.New("secret_access_key not found")
+		return nil, nil, nil, errors.New("secret_access_key not found")
 	}
 
 	if opt.Region == "" {
@@ -2129,25 +2162,36 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 		// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
 		awsSessionOpts.Config.Credentials = nil
 	}
+	// Setting this stops PutObject reading the body twice and seeking
+	// We add our own Content-MD5 for data protection
+	awsSessionOpts.Config.S3DisableContentMD5Validation = aws.Bool(true)
 	ses, err := session.NewSessionWithOptions(awsSessionOpts)
 	if err != nil {
-		return nil, nil, err
-	}
-	c := s3.New(ses)
-	if opt.V2Auth || opt.Region == "other-v2-signature" {
-		fs.Debugf(nil, "Using v2 auth")
-		signer := func(req *request.Request) {
-			// Ignore AnonymousCredentials object
-			if req.Config.Credentials == credentials.AnonymousCredentials {
-				return
+		return nil, nil, nil, err
+	}
+	newC := func(unsignedBody bool) *s3.S3 {
+		c := s3.New(ses)
+		if opt.V2Auth || opt.Region == "other-v2-signature" {
+			fs.Debugf(nil, "Using v2 auth")
+			signer := func(req *request.Request) {
+				// Ignore AnonymousCredentials object
+				if req.Config.Credentials == credentials.AnonymousCredentials {
+					return
+				}
+				sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
 			}
-			sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
+			c.Handlers.Sign.Clear()
+			c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+			c.Handlers.Sign.PushBack(signer)
+		} else if unsignedBody {
+			// If the body is unsigned then tell the signer that we aren't signing the payload
+			c.Handlers.Sign.Clear()
+			c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+			c.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler("v4.SignRequestHandler.WithUnsignedPayload", v4.WithUnsignedPayload))
 		}
-		c.Handlers.Sign.Clear()
-		c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
-		c.Handlers.Sign.PushBack(signer)
+		return c
 	}
-	return c, ses, nil
+	return newC(false), newC(true), ses, nil
 }
 
 func checkUploadChunkSize(cs fs.SizeSuffix) error {
@@ -2206,6 +2250,9 @@ func setQuirks(opt *Options) {
 		listObjectsV2 = false
 		virtualHostStyle = false
 		urlEncodeListings = false
+	case "Cloudflare":
+		virtualHostStyle = false
+		useMultipartEtag = false // currently multipart Etags are random
 	case "ArvanCloud":
 		listObjectsV2 = false
 		virtualHostStyle = false
@@ -2329,7 +2376,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
 	}
 	srv := getClient(ctx, opt)
-	c, ses, err := s3Connection(ctx, opt, srv)
+	c, cu, ses, err := s3Connection(ctx, opt, srv)
 	if err != nil {
 		return nil, err
 	}
@@ -2347,6 +2394,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ci:      ci,
 		ctx:     ctx,
 		c:       c,
+		cu:      cu,
 		ses:     ses,
 		pacer:   pc,
 		cache:   bucket.NewCache(),
@@ -2470,11 +2518,12 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
 	// Make a new session with the new region
 	oldRegion := f.opt.Region
 	f.opt.Region = region
-	c, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
+	c, cu, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
 	if err != nil {
 		return fmt.Errorf("creating new session failed: %w", err)
 	}
 	f.c = c
+	f.cu = cu
 	f.ses = ses
 
 	fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
@@ -3290,6 +3339,10 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 				st.Status = "Not an S3 object"
 				return
 			}
+			if o.storageClass != "GLACIER" && o.storageClass != "DEEP_ARCHIVE" {
+				st.Status = "Not GLACIER or DEEP_ARCHIVE storage class"
+				return
+			}
 			bucket, bucketPath := o.split()
 			reqCopy := req
 			reqCopy.Bucket = &bucket
@@ -3786,7 +3839,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	}
 
 	// calculate size of parts
-	partSize := int(f.opt.ChunkSize)
+	partSize := f.opt.ChunkSize
 
 	// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
 	// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
@@ -3797,11 +3850,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 				f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts))
 		})
 	} else {
-		// Adjust partSize until the number of parts is small enough.
-		if size/int64(partSize) >= uploadParts {
-			// Calculate partition size rounded up to the nearest MiB
-			partSize = int((((size / uploadParts) >> 20) + 1) << 20)
-		}
+		partSize = chunksize.Calculator(o, int(uploadParts), f.opt.ChunkSize)
 	}
 
 	memPool := f.getMemoryPool(int64(partSize))
@@ -3971,6 +4020,117 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	return etag, nil
 }
 
+// unWrapAwsError unwraps AWS errors, looking for a non AWS error
+//
+// It returns true if one was found and the error, or false and the
+// error passed in.
+func unWrapAwsError(err error) (found bool, outErr error) {
+	if awsErr, ok := err.(awserr.Error); ok {
+		var origErrs []error
+		if batchErr, ok := awsErr.(awserr.BatchError); ok {
+			origErrs = batchErr.OrigErrs()
+		} else {
+			origErrs = []error{awsErr.OrigErr()}
+		}
+		for _, origErr := range origErrs {
+			found, newErr := unWrapAwsError(origErr)
+			if found {
+				return found, newErr
+			}
+		}
+		return false, err
+	}
+	return true, err
+}
+
+// Upload a single part using PutObject
+func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
+	req.Body = readers.NewFakeSeeker(in, size)
+	var resp *s3.PutObjectOutput
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err = o.fs.cu.PutObject(req)
+		return o.fs.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		// Return the underlying error if we have a Serialization error if possible
+		//
+		// Serialization errors are synthesized locally in the SDK (not returned from the
+		// server). We'll get one if the SDK attempts a retry, however the FakeSeeker will
+		// remember the previous error from Read and return that.
+		if do, ok := err.(awserr.Error); ok && do.Code() == request.ErrCodeSerialization {
+			if found, newErr := unWrapAwsError(err); found {
+				err = newErr
+			}
+		}
+		return etag, lastModified, err
+	}
+	lastModified = time.Now()
+	etag = aws.StringValue(resp.ETag)
+	return etag, lastModified, nil
+}
+
+// Upload a single part using a presigned request
+func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
+	// Create the request
+	putObj, _ := o.fs.c.PutObjectRequest(req)
+
+	// Sign it so we can upload using a presigned request.
+	//
+	// Note the SDK didn't used to support streaming to
+	// PutObject so we used this work-around.
+	url, headers, err := putObj.PresignRequest(15 * time.Minute)
+	if err != nil {
+		return etag, lastModified, fmt.Errorf("s3 upload: sign request: %w", err)
+	}
+
+	if o.fs.opt.V2Auth && headers == nil {
+		headers = putObj.HTTPRequest.Header
+	}
+
+	// Set request to nil if empty so as not to make chunked encoding
+	if size == 0 {
+		in = nil
+	}
+
+	// create the vanilla http request
+	httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
+	if err != nil {
+		return etag, lastModified, fmt.Errorf("s3 upload: new request: %w", err)
+	}
+
+	// set the headers we signed and the length
+	httpReq.Header = headers
+	httpReq.ContentLength = size
+
+	var resp *http.Response
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		var err error
+		resp, err = o.fs.srv.Do(httpReq)
+		if err != nil {
+			return o.fs.shouldRetry(ctx, err)
+		}
+		body, err := rest.ReadBody(resp)
+		if err != nil {
+			return o.fs.shouldRetry(ctx, err)
+		}
+		if resp.StatusCode >= 200 && resp.StatusCode < 299 {
+			return false, nil
+		}
+		err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
+		return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	})
+	if err != nil {
+		return etag, lastModified, err
+	}
+	if resp != nil {
+		if date, err := http.ParseTime(resp.Header.Get("Date")); err != nil {
+			lastModified = date
+		}
+		etag = resp.Header.Get("Etag")
+	}
+	return etag, lastModified, nil
+}
+
 // Update the Object from in with modTime and size
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	bucket, bucketPath := o.split()
@@ -4022,6 +4182,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		ContentType: &mimeType,
 		Metadata:    metadata,
 	}
+	if size >= 0 {
+		req.ContentLength = &size
+	}
 	if md5sumBase64 != "" {
 		req.ContentMD5 = &md5sumBase64
 	}
@@ -4076,66 +4239,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
-	var resp *http.Response // response from PUT
-	var wantETag string     // Multipart upload Etag to check
+	var wantETag string        // Multipart upload Etag to check
+	var gotEtag string         // Etag we got from the upload
+	var lastModified time.Time // Time we got from the upload
 	if multipart {
 		wantETag, err = o.uploadMultipart(ctx, &req, size, in)
-		if err != nil {
-			return err
-		}
 	} else {
-
-		// Create the request
-		putObj, _ := o.fs.c.PutObjectRequest(&req)
-
-		// Sign it so we can upload using a presigned request.
-		//
-		// Note the SDK doesn't currently support streaming to
-		// PutObject so we'll use this work-around.
-		url, headers, err := putObj.PresignRequest(15 * time.Minute)
-		if err != nil {
-			return fmt.Errorf("s3 upload: sign request: %w", err)
-		}
-
-		if o.fs.opt.V2Auth && headers == nil {
-			headers = putObj.HTTPRequest.Header
-		}
-
-		// Set request to nil if empty so as not to make chunked encoding
-		if size == 0 {
-			in = nil
-		}
-
-		// create the vanilla http request
-		httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
-		if err != nil {
-			return fmt.Errorf("s3 upload: new request: %w", err)
-		}
-
-		// set the headers we signed and the length
-		httpReq.Header = headers
-		httpReq.ContentLength = size
-
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			var err error
-			resp, err = o.fs.srv.Do(httpReq)
-			if err != nil {
-				return o.fs.shouldRetry(ctx, err)
-			}
-			body, err := rest.ReadBody(resp)
-			if err != nil {
-				return o.fs.shouldRetry(ctx, err)
-			}
-			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
-				return false, nil
-			}
-			err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
-			return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-		})
-		if err != nil {
-			return err
+		if o.fs.opt.UsePresignedRequest {
+			gotEtag, lastModified, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
+		} else {
+			gotEtag, lastModified, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
 		}
 	}
+	if err != nil {
+		return err
+	}
 
 	// User requested we don't HEAD the object after uploading it
 	// so make up the object as best we can assuming it got
@@ -4148,11 +4266,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		o.mimeType = aws.StringValue(req.ContentType)
 		o.storageClass = aws.StringValue(req.StorageClass)
 		// If we have done a single part PUT request then we can read these
-		if resp != nil {
-			if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
-				o.lastModified = date
-			}
-			o.setMD5FromEtag(resp.Header.Get("Etag"))
+		if gotEtag != "" {
+			o.setMD5FromEtag(gotEtag)
+		}
+		if !o.lastModified.IsZero() {
+			o.lastModified = lastModified
 		}
 		return nil
 	}
diff --git a/backend/storj/fs.go b/backend/storj/fs.go
index da37748b5..95146edbf 100644
--- a/backend/storj/fs.go
+++ b/backend/storj/fs.go
@@ -579,7 +579,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		return nil, err
 	}
 
-	return newObjectFromUplink(f, "", upload.Info()), nil
+	return newObjectFromUplink(f, src.Remote(), upload.Info()), nil
 }
 
 // PutStream uploads to the remote path with the modTime given of indeterminate
diff --git a/docs/content/_index.md b/docs/content/_index.md
index 1f595763d..167c0749d 100644
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@@ -116,6 +116,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
 {{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
 {{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/s3/#scaleway" >}}
+{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
 {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 {{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 779541631..e41176809 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -588,3 +588,8 @@ put them back in again.` >}}
   * Lesmiscore <nao20010128@gmail.com>
   * ehsantdy <ehsan.tadayon@arvancloud.com>
   * SwazRGB <65694696+swazrgb@users.noreply.github.com>
+  * Mateusz Puczyński <mati6095@gmail.com>
+  * Michael C Tiernan - MIT-Research Computing Project <mtiernan@mit.edu>
+  * Kaspian <34658474+KaspianDev@users.noreply.github.com>
+  * Werner <EvilOlaf@users.noreply.github.com>
+  * Hugal31 <hugo.laloge@gmail.com>
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index 7107e3253..8ab466527 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -700,7 +700,7 @@ consider using the flag
 
 Google docs exist as virtual files on Google Drive and cannot be transferred
 to other filesystems natively. While it is possible to export a Google doc to
-a normal file (with `.xlsx` extension, for example), it's not possible
+a normal file (with `.xlsx` extension, for example), it is not possible
 to import a normal file back into a Google document.
 
 Bisync's handling of Google Doc files is to flag them in the run log output
diff --git a/docs/content/faq.md b/docs/content/faq.md
index f6f31a1b6..ecd240882 100644
--- a/docs/content/faq.md
+++ b/docs/content/faq.md
@@ -31,7 +31,7 @@ the node running rclone would need to have lots of bandwidth.
 
 The syncs would be incremental (on a file by file basis).
 
-Eg
+e.g.
 
     rclone sync -i drive:Folder s3:bucket
 
diff --git a/docs/content/install.md b/docs/content/install.md
index 52e63f534..eccea9543 100644
--- a/docs/content/install.md
+++ b/docs/content/install.md
@@ -26,11 +26,11 @@ using the [rclone selfupdate](/commands/rclone_selfupdate/) command.
 
 To install rclone on Linux/macOS/BSD systems, run:
 
-    curl https://rclone.org/install.sh | sudo bash
+    sudo -v ; curl https://rclone.org/install.sh | sudo bash
 
 For beta installation, run:
 
-    curl https://rclone.org/install.sh | sudo bash -s beta
+    sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
 
 Note that this script checks the version of rclone installed first and
 won't re-download if not needed.
diff --git a/docs/content/install.sh b/docs/content/install.sh
index 1b854535a..8a8242fee 100755
--- a/docs/content/install.sh
+++ b/docs/content/install.sh
@@ -12,7 +12,7 @@ set -e
 #when adding a tool to the list make sure to also add its corresponding command further in the script
 unzip_tools_list=('unzip' '7z' 'busybox')
 
-usage() { echo "Usage: curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
+usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
 
 #check for beta flag
 if [ -n "$1" ] && [ "$1" != "beta" ]; then
diff --git a/docs/content/s3.md b/docs/content/s3.md
index f51f4b4a1..aaf76804d 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -12,6 +12,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
 {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
 {{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
+{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
 {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
 {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@@ -2479,12 +2480,19 @@ You will be able to list and copy data but not upload it.
 This is the provider used as main example and described in the [configuration](#configuration) section above.
 
 ### AWS Snowball Edge
-[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware appliance used for transferring
-bulk data back to AWS. Its main software interface is S3 object storage.
 
-To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service'
-be sure to set `upload_cutoff = 0` otherwise you will run into authentication header issues as
-the snowball device does not support query parameter based authentication.
+[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware
+appliance used for transferring bulk data back to AWS. Its main
+software interface is S3 object storage.
+
+To use rclone with AWS Snowball Edge devices, configure as standard
+for an 'S3 Compatible Service'.
+
+If using rclone pre v1.59 be sure to set `upload_cutoff = 0` otherwise
+you will run into authentication header issues as the snowball device
+does not support query parameter based authentication.
+
+With rclone v1.59 or later setting `upload_cutoff` should not be necessary.
 
 eg.
 ```
@@ -2523,10 +2531,11 @@ server_side_encryption =
 storage_class =
 ```
 
-If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you
-may need to supply the parameter `--s3-upload-cutoff 0` or put this in
-the config file as `upload_cutoff 0` to work around a bug which causes
-uploading of small files to fail.
+If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a
+version of rclone before v1.59 then you may need to supply the
+parameter `--s3-upload-cutoff 0` or put this in the config file as
+`upload_cutoff 0` to work around a bug which causes uploading of small
+files to fail.
 
 Note also that Ceph sometimes puts `/` in the passwords it gives
 users.  If you read the secret access key using the command line tools
@@ -2553,6 +2562,115 @@ removed).
 Because this is a json dump, it is encoding the `/` as `\/`, so if you
 use the secret key as `xxxxxx/xxxx`  it will work fine.
 
+### Cloudflare R2 {#cloudflare-r2}
+
+[Cloudflare R2](https://blog.cloudflare.com/r2-open-beta/) Storage
+allows developers to store large amounts of unstructured data without
+the costly egress bandwidth fees associated with typical cloud storage
+services.
+
+Here is an example of making a Cloudflare R2 configuration. First run:
+
+    rclone config
+
+This will guide you through an interactive setup process.
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> r2
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+...
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+...
+Storage> s3
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+...
+XX / Cloudflare R2 Storage
+   \ (Cloudflare)
+...
+provider> Cloudflare
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth> 1
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> ACCESS_KEY
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> SECRET_ACCESS_KEY
+Option region.
+Region to connect to.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
+   \ (auto)
+region> 1
+Option endpoint.
+Endpoint for S3 API.
+Required when using an S3 clone.
+Enter a value. Press Enter to leave empty.
+endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+...
+acl> 1
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+This will leave your config looking something like:
+
+```
+[r2]
+type = s3
+provider = Cloudflare
+access_key_id = ACCESS_KEY
+secret_access_key = SECRET_ACCESS_KEY
+region = auto
+endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com
+acl = private
+```
+
+Now run `rclone lsf r2:` to see your buckets and `rclone lsf
+r2:bucket` to look within a bucket.
+
 ### Dreamhost
 
 Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is
diff --git a/fs/chunksize/chunksize.go b/fs/chunksize/chunksize.go
new file mode 100644
index 000000000..ab171f07d
--- /dev/null
+++ b/fs/chunksize/chunksize.go
@@ -0,0 +1,36 @@
+// Package chunksize calculates a suitable chunk size for large uploads
+package chunksize
+
+import (
+	"github.com/rclone/rclone/fs"
+)
+
+/*
+ Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi
+
+ For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest
+ possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption
+ that some backends may only allow integer type parameters when specifying the chunk size.
+
+ Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the
+ smallest chunk size necessary to allow the upload to proceed.
+*/
+func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix {
+	fileSize := fs.SizeSuffix(objInfo.Size())
+	requiredChunks := fileSize / defaultChunkSize
+	if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) {
+		return defaultChunkSize
+	}
+
+	minChunk := fileSize / fs.SizeSuffix(maxParts)
+	remainder := minChunk % fs.Mebi
+	if remainder != 0 {
+		minChunk += fs.Mebi - remainder
+	}
+	if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB
+		minChunk += fs.Mebi
+	}
+
+	fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk)
+	return minChunk
+}
diff --git a/fs/mount_helper.go b/fs/mount_helper.go
index 884b1f67c..f45b85895 100644
--- a/fs/mount_helper.go
+++ b/fs/mount_helper.go
@@ -208,7 +208,7 @@ func parseHelperOptionString(optString string) (opts []string, err error) {
 				if len(param) == 0 {
 					return nil, errHelperEmptyOption
 				}
-				if param[0] == '-' || param[0] == '_' {
+				if param[0] == '-' {
 					return nil, errHelperOptionName
 				}
 				prev = i + 1
diff --git a/fs/open_options.go b/fs/open_options.go
index 8d997118a..84e43045c 100644
--- a/fs/open_options.go
+++ b/fs/open_options.go
@@ -138,8 +138,14 @@ func (o *RangeOption) Decode(size int64) (offset, limit int64) {
 // absolute fetch using the size passed in and makes sure the range does
 // not exceed filesize. Some remotes (e.g. Onedrive, Box) don't support
 // range requests which index from the end.
+//
+// It also adjusts any SeekOption~s, turning them into absolute
+// RangeOption~s instead.
 func FixRangeOption(options []OpenOption, size int64) {
-	if size == 0 {
+	if size < 0 {
+		// Can't do anything for unknown length objects
+		return
+	} else if size == 0 {
 		// if size 0 then remove RangeOption~s
 		// replacing with a NullOptions~s which won't be rendered
 		for i := range options {
@@ -150,18 +156,21 @@ func FixRangeOption(options []OpenOption, size int64) {
 		}
 		return
 	}
-	for i := range options {
-		option := options[i]
-		if x, ok := option.(*RangeOption); ok {
+	for i, option := range options {
+		switch x := option.(type) {
+		case *RangeOption:
 			// If start is < 0 then fetch from the end
 			if x.Start < 0 {
 				x = &RangeOption{Start: size - x.End, End: -1}
 				options[i] = x
 			}
-			if x.End > size {
+			// If end is too big or undefined, fetch to the end
+			if x.End > size || x.End < 0 {
 				x = &RangeOption{Start: x.Start, End: size - 1}
 				options[i] = x
 			}
+		case *SeekOption:
+			options[i] = &RangeOption{Start: x.Offset, End: size - 1}
 		}
 	}
 }
diff --git a/lib/readers/fakeseeker.go b/lib/readers/fakeseeker.go
new file mode 100644
index 000000000..844fc0422
--- /dev/null
+++ b/lib/readers/fakeseeker.go
@@ -0,0 +1,72 @@
+package readers
+
+import (
+	"errors"
+	"fmt"
+	"io"
+)
+
+// FakeSeeker adapts an io.Seeker into an io.ReadSeeker
+type FakeSeeker struct {
+	in      io.Reader
+	readErr error
+	length  int64
+	offset  int64
+	read    bool
+}
+
+// NewFakeSeeker creates a fake io.ReadSeeker from an io.Reader
+//
+// This can be seeked before reading to discover the length passed in.
+func NewFakeSeeker(in io.Reader, length int64) io.ReadSeeker {
+	if rs, ok := in.(io.ReadSeeker); ok {
+		return rs
+	}
+	return &FakeSeeker{
+		in:     in,
+		length: length,
+	}
+}
+
+// Seek the stream - possible only before reading
+func (r *FakeSeeker) Seek(offset int64, whence int) (abs int64, err error) {
+	if r.readErr != nil {
+		return 0, r.readErr
+	}
+	if r.read {
+		return 0, fmt.Errorf("FakeSeeker: can't Seek(%d, %d) after reading", offset, whence)
+	}
+	switch whence {
+	case io.SeekStart:
+		abs = offset
+	case io.SeekCurrent:
+		abs = r.offset + offset
+	case io.SeekEnd:
+		abs = r.length + offset
+	default:
+		return 0, errors.New("FakeSeeker: invalid whence")
+	}
+	if abs < 0 {
+		return 0, errors.New("FakeSeeker: negative position")
+	}
+	r.offset = abs
+	return abs, nil
+}
+
+// Read data from the stream. Will give an error if seeked.
+func (r *FakeSeeker) Read(p []byte) (n int, err error) {
+	if r.readErr != nil {
+		return 0, r.readErr
+	}
+	if !r.read && r.offset != 0 {
+		return 0, errors.New("FakeSeeker: not at start: can't read")
+	}
+	n, err = r.in.Read(p)
+	if n != 0 {
+		r.read = true
+	}
+	if err != nil {
+		r.readErr = err
+	}
+	return n, err
+}


Test patch:
diff --git a/fs/chunksize/chunksize_test.go b/fs/chunksize/chunksize_test.go
new file mode 100644
index 000000000..9cdeb70d6
--- /dev/null
+++ b/fs/chunksize/chunksize_test.go
@@ -0,0 +1,40 @@
+package chunksize
+
+import (
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/object"
+)
+
+func TestComputeChunkSize(t *testing.T) {
+	tests := map[string]struct {
+		fileSize         fs.SizeSuffix
+		maxParts         int
+		defaultChunkSize fs.SizeSuffix
+		expected         fs.SizeSuffix
+	}{
+		"default size returned when file size is small enough":             {fileSize: 1000, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
+		"default size returned when file size is just 1 byte small enough": {fileSize: toSizeSuffixMiB(100000) - 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
+		"no rounding up when everything divides evenly":                    {fileSize: toSizeSuffixMiB(1000000), maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(100)},
+		"rounding up to nearest MiB when not quite enough parts":           {fileSize: toSizeSuffixMiB(1000000), maxParts: 9999, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
+		"rounding up to nearest MiB when one extra byte":                   {fileSize: toSizeSuffixMiB(1000000) + 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
+		"expected MiB value when rounding sets to absolute minimum":        {fileSize: toSizeSuffixMiB(1) - 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(1)},
+		"expected MiB value when rounding to absolute min with extra":      {fileSize: toSizeSuffixMiB(1) + 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(2)},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			src := object.NewStaticObjectInfo("mock", time.Now(), int64(tc.fileSize), true, nil, nil)
+			result := Calculator(src, tc.maxParts, tc.defaultChunkSize)
+			if result != tc.expected {
+				t.Fatalf("expected: %v, got: %v", tc.expected, result)
+			}
+		})
+	}
+}
+
+func toSizeSuffixMiB(size int64) fs.SizeSuffix {
+	return fs.SizeSuffix(size * int64(fs.Mebi))
+}
diff --git a/fs/mount_helper_test.go b/fs/mount_helper_test.go
index 00c4fba79..3235b1195 100644
--- a/fs/mount_helper_test.go
+++ b/fs/mount_helper_test.go
@@ -20,7 +20,7 @@ func TestMountHelperArgs(t *testing.T) {
 		src: []string{},
 		dst: []string{"mount", "--daemon"},
 	}, {
-		src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env`},
+		src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env,_netdev`},
 		dst: []string{"mount", "--read-only", "--verbose=3", "--daemon"},
 		env: "HTTPS_PROXY=a b;c,d?EF",
 	}}
diff --git a/fs/open_options_test.go b/fs/open_options_test.go
index 52fc42ddb..829f45c19 100644
--- a/fs/open_options_test.go
+++ b/fs/open_options_test.go
@@ -149,6 +149,16 @@ func TestFixRangeOptions(t *testing.T) {
 			in:   []OpenOption{},
 			want: []OpenOption{},
 		},
+		{
+			name: "Unknown size -1",
+			in: []OpenOption{
+				&RangeOption{Start: 1, End: -1},
+			},
+			want: []OpenOption{
+				&RangeOption{Start: 1, End: -1},
+			},
+			size: -1,
+		},
 		{
 			name: "Fetch a range with size=0",
 			in: []OpenOption{
@@ -183,7 +193,7 @@ func TestFixRangeOptions(t *testing.T) {
 				&RangeOption{Start: 1, End: -1},
 			},
 			want: []OpenOption{
-				&RangeOption{Start: 1, End: -1},
+				&RangeOption{Start: 1, End: 99},
 			},
 			size: 100,
 		},
@@ -193,7 +203,7 @@ func TestFixRangeOptions(t *testing.T) {
 				&RangeOption{Start: -1, End: 10},
 			},
 			want: []OpenOption{
-				&RangeOption{Start: 90, End: -1},
+				&RangeOption{Start: 90, End: 99},
 			},
 			size: 100,
 		},
@@ -207,6 +217,20 @@ func TestFixRangeOptions(t *testing.T) {
 			},
 			size: 100,
 		},
+		{
+			name: "SeekOption",
+			in: []OpenOption{
+				&HTTPOption{Key: "a", Value: "1"},
+				&SeekOption{Offset: 10},
+				&HTTPOption{Key: "b", Value: "2"},
+			},
+			want: []OpenOption{
+				&HTTPOption{Key: "a", Value: "1"},
+				&RangeOption{Start: 10, End: 99},
+				&HTTPOption{Key: "b", Value: "2"},
+			},
+			size: 100,
+		},
 	} {
 		FixRangeOption(test.in, test.size)
 		assert.Equal(t, test.want, test.in, test.name)
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index ca36aa7d8..1b6bb9e1d 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -134,7 +134,7 @@ backends:
    remote:   "TestHubic:"
    fastlist: false
  - backend:  "internetarchive"
-   remote:   "TestIA:lesmi-rclone-test/"
+   remote:   "TestIA:rclone-integration-test"
    fastlist: true
  - backend:  "jottacloud"
    remote:   "TestJottacloud:"
@@ -204,6 +204,11 @@ backends:
  - backend:  "s3"
    remote:   "TestS3Alibaba:"
    fastlist: true
+ - backend:  "s3"
+   remote:   "TestS3R2:"
+   fastlist: true
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/SetTier
  - backend:  "sftp"
    remote:   "TestSFTPOpenssh:"
    fastlist: false
diff --git a/lib/readers/fakeseeker_test.go b/lib/readers/fakeseeker_test.go
new file mode 100644
index 000000000..0702b7cbb
--- /dev/null
+++ b/lib/readers/fakeseeker_test.go
@@ -0,0 +1,87 @@
+package readers
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Check interface
+var _ io.ReadSeeker = &FakeSeeker{}
+
+func TestFakeSeeker(t *testing.T) {
+	// Test that passing in an io.ReadSeeker just passes it through
+	bufReader := bytes.NewReader([]byte{1})
+	r := NewFakeSeeker(bufReader, 5)
+	assert.Equal(t, r, bufReader)
+
+	in := bytes.NewBufferString("hello")
+	buf := make([]byte, 16)
+	r = NewFakeSeeker(in, 5)
+	assert.NotEqual(t, r, in)
+
+	// check the seek offset is as passed in
+	checkPos := func(pos int64) {
+		abs, err := r.Seek(0, io.SeekCurrent)
+		require.NoError(t, err)
+		assert.Equal(t, pos, abs)
+	}
+
+	// Test some seeking
+	checkPos(0)
+
+	abs, err := r.Seek(2, io.SeekStart)
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), abs)
+	checkPos(2)
+
+	abs, err = r.Seek(-1, io.SeekEnd)
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), abs)
+	checkPos(4)
+
+	// Check can't read if not at start
+	_, err = r.Read(buf)
+	require.ErrorContains(t, err, "not at start")
+
+	// Seek back to start
+	abs, err = r.Seek(-4, io.SeekCurrent)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), abs)
+	checkPos(0)
+
+	_, err = r.Seek(42, 17)
+	require.ErrorContains(t, err, "invalid whence")
+
+	_, err = r.Seek(-1, io.SeekStart)
+	require.ErrorContains(t, err, "negative position")
+
+	// Test reading now seeked back to the start
+	n, err := r.Read(buf)
+	require.NoError(t, err)
+	assert.Equal(t, 5, n)
+	assert.Equal(t, []byte("hello"), buf[:5])
+
+	// Seeking should give an error now
+	_, err = r.Seek(-1, io.SeekEnd)
+	require.ErrorContains(t, err, "after reading")
+}
+
+func TestFakeSeekerError(t *testing.T) {
+	in := bytes.NewBufferString("hello")
+	r := NewFakeSeeker(in, 5)
+	assert.NotEqual(t, r, in)
+
+	buf, err := io.ReadAll(r)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("hello"), buf)
+
+	_, err = r.Read(buf)
+	assert.Equal(t, io.EOF, err)
+
+	_, err = r.Seek(0, io.SeekStart)
+	assert.Equal(t, io.EOF, err)
+}