Repository: rclone/rclone
Pull request #6147 (2022-05-04 12:39:46)

Pre verify sudo authorization "-v" before calling curl.
	The way this was, the curl interferes with the
	sudo and bash invocation. Here it will authenticate first.


#### What is the purpose of this change?

<!--
Describe the changes here
-->

Documentation "correction" showing the sequence of "verify sudo" then "use sudo" before any other commands would interfere with the input/output

#### Was the change discussed in an issue or in the forum before?

<!--
Link issues and relevant forum posts here.
-->
No.

#### Checklist

- [ N ] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [ N/A ] I have added tests for all changes in this PR if appropriate.
- [ N/A ] I have added documentation for the changes if appropriate.
- [ ?" ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [ N ] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/README.md b/README.md
index 1f3ca3dfb..0235323b5 100644
--- a/README.md
+++ b/README.md
@@ -29,6 +29,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
   * China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
+  * Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
   * Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 81f071b92..d8ba92724 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -32,6 +32,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/ncw/swift/v2"
 	"github.com/rclone/rclone/fs"
@@ -58,7 +59,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Options: []fs.Option{{
@@ -78,6 +79,9 @@ func init() {
 			}, {
 				Value: "ChinaMobile",
 				Help:  "China Mobile Ecloud Elastic Object Storage (EOS)",
+			}, {
+				Value: "Cloudflare",
+				Help:  "Cloudflare R2 Storage",
 			}, {
 				Value: "ArvanCloud",
 				Help:  "Arvan Cloud Object Storage (AOS)",
@@ -297,10 +301,18 @@ func init() {
 				Value: "fr-par",
 				Help:  "Paris, France",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region to connect to.",
+			Provider: "Cloudflare",
+			Examples: []fs.OptionExample{{
+				Value: "auto",
+				Help:  "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
+			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -1270,7 +1282,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
+			Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1823,6 +1835,20 @@ This should be true, false or left unset to use the default for the provider.
 `,
 			Default:  fs.Tristate{},
 			Advanced: true,
+		}, {
+			Name: "use_presigned_request",
+			Help: `Whether to use a presigned request or PutObject for single part uploads
+
+If this is false rclone will use PutObject from the AWS SDK to upload
+an object.
+
+Versions of rclone < 1.59 use presigned requests to upload a single
+part object and setting this flag to true will re-enable that
+functionality. This shouldn't be necessary except in exceptional
+circumstances or for testing.
+`,
+			Default:  false,
+			Advanced: true,
 		},
 		}})
 }
@@ -1888,6 +1914,7 @@ type Options struct {
 	DisableHTTP2          bool                 `config:"disable_http2"`
 	DownloadURL           string               `config:"download_url"`
 	UseMultipartEtag      fs.Tristate          `config:"use_multipart_etag"`
+	UsePresignedRequest   bool                 `config:"use_presigned_request"`
 }
 
 // Fs represents a remote s3 server
@@ -1899,6 +1926,7 @@ type Fs struct {
 	ctx           context.Context  // global context for reading config
 	features      *fs.Features     // optional features
 	c             *s3.S3           // the connection to the s3 server
+	cu            *s3.S3           // unsigned connection to the s3 server for PutObject
 	ses           *session.Session // the s3 session
 	rootBucket    string           // bucket part of root (if any)
 	rootDirectory string           // directory part of root (if any)
@@ -2031,7 +2059,11 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 }
 
 // s3Connection makes a connection to s3
-func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
+//
+// If unsignedBody is set then the connection is configured for
+// unsigned bodies which is necessary for PutObject if we don't want
+// it to seek
+func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *s3.S3, *session.Session, error) {
 	ci := fs.GetConfig(ctx)
 	// Make the auth
 	v := credentials.Value{
@@ -2048,7 +2080,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 	// start a new AWS session
 	awsSession, err := session.NewSession()
 	if err != nil {
-		return nil, nil, fmt.Errorf("NewSession: %w", err)
+		return nil, nil, nil, fmt.Errorf("NewSession: %w", err)
 	}
 
 	// first provider to supply a credential set "wins"
@@ -2088,9 +2120,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 		// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
 		cred = credentials.AnonymousCredentials
 	case v.AccessKeyID == "":
-		return nil, nil, errors.New("access_key_id not found")
+		return nil, nil, nil, errors.New("access_key_id not found")
 	case v.SecretAccessKey == "":
-		return nil, nil, errors.New("secret_access_key not found")
+		return nil, nil, nil, errors.New("secret_access_key not found")
 	}
 
 	if opt.Region == "" {
@@ -2129,25 +2161,36 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 		// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
 		awsSessionOpts.Config.Credentials = nil
 	}
+	// Setting this stops PutObject reading the body twice and seeking
+	// We add our own Content-MD5 for data protection
+	awsSessionOpts.Config.S3DisableContentMD5Validation = aws.Bool(true)
 	ses, err := session.NewSessionWithOptions(awsSessionOpts)
 	if err != nil {
-		return nil, nil, err
-	}
-	c := s3.New(ses)
-	if opt.V2Auth || opt.Region == "other-v2-signature" {
-		fs.Debugf(nil, "Using v2 auth")
-		signer := func(req *request.Request) {
-			// Ignore AnonymousCredentials object
-			if req.Config.Credentials == credentials.AnonymousCredentials {
-				return
+		return nil, nil, nil, err
+	}
+	newC := func(unsignedBody bool) *s3.S3 {
+		c := s3.New(ses)
+		if opt.V2Auth || opt.Region == "other-v2-signature" {
+			fs.Debugf(nil, "Using v2 auth")
+			signer := func(req *request.Request) {
+				// Ignore AnonymousCredentials object
+				if req.Config.Credentials == credentials.AnonymousCredentials {
+					return
+				}
+				sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
 			}
-			sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
+			c.Handlers.Sign.Clear()
+			c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+			c.Handlers.Sign.PushBack(signer)
+		} else if unsignedBody {
+			// If the body is unsigned then tell the signer that we aren't signing the payload
+			c.Handlers.Sign.Clear()
+			c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+			c.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler("v4.SignRequestHandler.WithUnsignedPayload", v4.WithUnsignedPayload))
 		}
-		c.Handlers.Sign.Clear()
-		c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
-		c.Handlers.Sign.PushBack(signer)
+		return c
 	}
-	return c, ses, nil
+	return newC(false), newC(true), ses, nil
 }
 
 func checkUploadChunkSize(cs fs.SizeSuffix) error {
@@ -2206,6 +2249,9 @@ func setQuirks(opt *Options) {
 		listObjectsV2 = false
 		virtualHostStyle = false
 		urlEncodeListings = false
+	case "Cloudflare":
+		virtualHostStyle = false
+		useMultipartEtag = false // currently multipart Etags are random
 	case "ArvanCloud":
 		listObjectsV2 = false
 		virtualHostStyle = false
@@ -2329,7 +2375,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
 	}
 	srv := getClient(ctx, opt)
-	c, ses, err := s3Connection(ctx, opt, srv)
+	c, cu, ses, err := s3Connection(ctx, opt, srv)
 	if err != nil {
 		return nil, err
 	}
@@ -2347,6 +2393,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ci:      ci,
 		ctx:     ctx,
 		c:       c,
+		cu:      cu,
 		ses:     ses,
 		pacer:   pc,
 		cache:   bucket.NewCache(),
@@ -2470,11 +2517,12 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
 	// Make a new session with the new region
 	oldRegion := f.opt.Region
 	f.opt.Region = region
-	c, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
+	c, cu, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
 	if err != nil {
 		return fmt.Errorf("creating new session failed: %w", err)
 	}
 	f.c = c
+	f.cu = cu
 	f.ses = ses
 
 	fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
@@ -3971,6 +4019,117 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	return etag, nil
 }
 
+// unWrapAwsError unwraps AWS errors, looking for a non AWS error
+//
+// It returns true if one was found and the error, or false and the
+// error passed in.
+func unWrapAwsError(err error) (found bool, outErr error) {
+	if awsErr, ok := err.(awserr.Error); ok {
+		var origErrs []error
+		if batchErr, ok := awsErr.(awserr.BatchError); ok {
+			origErrs = batchErr.OrigErrs()
+		} else {
+			origErrs = []error{awsErr.OrigErr()}
+		}
+		for _, origErr := range origErrs {
+			found, newErr := unWrapAwsError(origErr)
+			if found {
+				return found, newErr
+			}
+		}
+		return false, err
+	}
+	return true, err
+}
+
+// Upload a single part using PutObject
+func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
+	req.Body = readers.NewFakeSeeker(in, size)
+	var resp *s3.PutObjectOutput
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		resp, err = o.fs.cu.PutObject(req)
+		return o.fs.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		// Return the underlying error if we have a Serialization error if possible
+		//
+		// Serialization errors are synthesized locally in the SDK (not returned from the
+		// server). We'll get one if the SDK attempts a retry, however the FakeSeeker will
+		// remember the previous error from Read and return that.
+		if do, ok := err.(awserr.Error); ok && do.Code() == request.ErrCodeSerialization {
+			if found, newErr := unWrapAwsError(err); found {
+				err = newErr
+			}
+		}
+		return etag, lastModified, err
+	}
+	lastModified = time.Now()
+	etag = aws.StringValue(resp.ETag)
+	return etag, lastModified, nil
+}
+
+// Upload a single part using a presigned request
+func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
+	// Create the request
+	putObj, _ := o.fs.c.PutObjectRequest(req)
+
+	// Sign it so we can upload using a presigned request.
+	//
+	// Note the SDK didn't used to support streaming to
+	// PutObject so we used this work-around.
+	url, headers, err := putObj.PresignRequest(15 * time.Minute)
+	if err != nil {
+		return etag, lastModified, fmt.Errorf("s3 upload: sign request: %w", err)
+	}
+
+	if o.fs.opt.V2Auth && headers == nil {
+		headers = putObj.HTTPRequest.Header
+	}
+
+	// Set request to nil if empty so as not to make chunked encoding
+	if size == 0 {
+		in = nil
+	}
+
+	// create the vanilla http request
+	httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
+	if err != nil {
+		return etag, lastModified, fmt.Errorf("s3 upload: new request: %w", err)
+	}
+
+	// set the headers we signed and the length
+	httpReq.Header = headers
+	httpReq.ContentLength = size
+
+	var resp *http.Response
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		var err error
+		resp, err = o.fs.srv.Do(httpReq)
+		if err != nil {
+			return o.fs.shouldRetry(ctx, err)
+		}
+		body, err := rest.ReadBody(resp)
+		if err != nil {
+			return o.fs.shouldRetry(ctx, err)
+		}
+		if resp.StatusCode >= 200 && resp.StatusCode < 299 {
+			return false, nil
+		}
+		err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
+		return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	})
+	if err != nil {
+		return etag, lastModified, err
+	}
+	if resp != nil {
+		if date, err := http.ParseTime(resp.Header.Get("Date")); err != nil {
+			lastModified = date
+		}
+		etag = resp.Header.Get("Etag")
+	}
+	return etag, lastModified, nil
+}
+
 // Update the Object from in with modTime and size
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	bucket, bucketPath := o.split()
@@ -4022,6 +4181,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		ContentType: &mimeType,
 		Metadata:    metadata,
 	}
+	if size >= 0 {
+		req.ContentLength = &size
+	}
 	if md5sumBase64 != "" {
 		req.ContentMD5 = &md5sumBase64
 	}
@@ -4076,66 +4238,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
-	var resp *http.Response // response from PUT
-	var wantETag string     // Multipart upload Etag to check
+	var wantETag string        // Multipart upload Etag to check
+	var gotEtag string         // Etag we got from the upload
+	var lastModified time.Time // Time we got from the upload
 	if multipart {
 		wantETag, err = o.uploadMultipart(ctx, &req, size, in)
-		if err != nil {
-			return err
-		}
 	} else {
-
-		// Create the request
-		putObj, _ := o.fs.c.PutObjectRequest(&req)
-
-		// Sign it so we can upload using a presigned request.
-		//
-		// Note the SDK doesn't currently support streaming to
-		// PutObject so we'll use this work-around.
-		url, headers, err := putObj.PresignRequest(15 * time.Minute)
-		if err != nil {
-			return fmt.Errorf("s3 upload: sign request: %w", err)
-		}
-
-		if o.fs.opt.V2Auth && headers == nil {
-			headers = putObj.HTTPRequest.Header
-		}
-
-		// Set request to nil if empty so as not to make chunked encoding
-		if size == 0 {
-			in = nil
-		}
-
-		// create the vanilla http request
-		httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
-		if err != nil {
-			return fmt.Errorf("s3 upload: new request: %w", err)
-		}
-
-		// set the headers we signed and the length
-		httpReq.Header = headers
-		httpReq.ContentLength = size
-
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			var err error
-			resp, err = o.fs.srv.Do(httpReq)
-			if err != nil {
-				return o.fs.shouldRetry(ctx, err)
-			}
-			body, err := rest.ReadBody(resp)
-			if err != nil {
-				return o.fs.shouldRetry(ctx, err)
-			}
-			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
-				return false, nil
-			}
-			err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
-			return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
-		})
-		if err != nil {
-			return err
+		if o.fs.opt.UsePresignedRequest {
+			gotEtag, lastModified, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
+		} else {
+			gotEtag, lastModified, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
 		}
 	}
+	if err != nil {
+		return err
+	}
 
 	// User requested we don't HEAD the object after uploading it
 	// so make up the object as best we can assuming it got
@@ -4148,11 +4265,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		o.mimeType = aws.StringValue(req.ContentType)
 		o.storageClass = aws.StringValue(req.StorageClass)
 		// If we have done a single part PUT request then we can read these
-		if resp != nil {
-			if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
-				o.lastModified = date
-			}
-			o.setMD5FromEtag(resp.Header.Get("Etag"))
+		if gotEtag != "" {
+			o.setMD5FromEtag(gotEtag)
+		}
+		if !o.lastModified.IsZero() {
+			o.lastModified = lastModified
 		}
 		return nil
 	}
diff --git a/docs/content/_index.md b/docs/content/_index.md
index 1f595763d..167c0749d 100644
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@@ -116,6 +116,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
 {{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
 {{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/s3/#scaleway" >}}
+{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
 {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 {{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 779541631..2ebd2d192 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -588,3 +588,5 @@ put them back in again.` >}}
   * Lesmiscore <nao20010128@gmail.com>
   * ehsantdy <ehsan.tadayon@arvancloud.com>
   * SwazRGB <65694696+swazrgb@users.noreply.github.com>
+  * Mateusz Puczyński <mati6095@gmail.com>
+  * Michael C Tiernan - MIT-Research Computing Project <mtiernan@mit.edu>
diff --git a/docs/content/install.md b/docs/content/install.md
index 52e63f534..eccea9543 100644
--- a/docs/content/install.md
+++ b/docs/content/install.md
@@ -26,11 +26,11 @@ using the [rclone selfupdate](/commands/rclone_selfupdate/) command.
 
 To install rclone on Linux/macOS/BSD systems, run:
 
-    curl https://rclone.org/install.sh | sudo bash
+    sudo -v ; curl https://rclone.org/install.sh | sudo bash
 
 For beta installation, run:
 
-    curl https://rclone.org/install.sh | sudo bash -s beta
+    sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
 
 Note that this script checks the version of rclone installed first and
 won't re-download if not needed.
diff --git a/docs/content/install.sh b/docs/content/install.sh
index 1b854535a..8a8242fee 100755
--- a/docs/content/install.sh
+++ b/docs/content/install.sh
@@ -12,7 +12,7 @@ set -e
 #when adding a tool to the list make sure to also add its corresponding command further in the script
 unzip_tools_list=('unzip' '7z' 'busybox')
 
-usage() { echo "Usage: curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
+usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
 
 #check for beta flag
 if [ -n "$1" ] && [ "$1" != "beta" ]; then
diff --git a/docs/content/s3.md b/docs/content/s3.md
index f51f4b4a1..aaf76804d 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -12,6 +12,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
 {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
 {{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
+{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
 {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
 {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
 {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@@ -2479,12 +2480,19 @@ You will be able to list and copy data but not upload it.
 This is the provider used as main example and described in the [configuration](#configuration) section above.
 
 ### AWS Snowball Edge
-[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware appliance used for transferring
-bulk data back to AWS. Its main software interface is S3 object storage.
 
-To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service'
-be sure to set `upload_cutoff = 0` otherwise you will run into authentication header issues as
-the snowball device does not support query parameter based authentication.
+[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware
+appliance used for transferring bulk data back to AWS. Its main
+software interface is S3 object storage.
+
+To use rclone with AWS Snowball Edge devices, configure as standard
+for an 'S3 Compatible Service'.
+
+If using rclone pre v1.59 be sure to set `upload_cutoff = 0` otherwise
+you will run into authentication header issues as the snowball device
+does not support query parameter based authentication.
+
+With rclone v1.59 or later setting `upload_cutoff` should not be necessary.
 
 eg.
 ```
@@ -2523,10 +2531,11 @@ server_side_encryption =
 storage_class =
 ```
 
-If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you
-may need to supply the parameter `--s3-upload-cutoff 0` or put this in
-the config file as `upload_cutoff 0` to work around a bug which causes
-uploading of small files to fail.
+If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a
+version of rclone before v1.59 then you may need to supply the
+parameter `--s3-upload-cutoff 0` or put this in the config file as
+`upload_cutoff 0` to work around a bug which causes uploading of small
+files to fail.
 
 Note also that Ceph sometimes puts `/` in the passwords it gives
 users.  If you read the secret access key using the command line tools
@@ -2553,6 +2562,115 @@ removed).
 Because this is a json dump, it is encoding the `/` as `\/`, so if you
 use the secret key as `xxxxxx/xxxx`  it will work fine.
 
+### Cloudflare R2 {#cloudflare-r2}
+
+[Cloudflare R2](https://blog.cloudflare.com/r2-open-beta/) Storage
+allows developers to store large amounts of unstructured data without
+the costly egress bandwidth fees associated with typical cloud storage
+services.
+
+Here is an example of making a Cloudflare R2 configuration. First run:
+
+    rclone config
+
+This will guide you through an interactive setup process.
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> r2
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+...
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+...
+Storage> s3
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+...
+XX / Cloudflare R2 Storage
+   \ (Cloudflare)
+...
+provider> Cloudflare
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth> 1
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> ACCESS_KEY
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> SECRET_ACCESS_KEY
+Option region.
+Region to connect to.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
+   \ (auto)
+region> 1
+Option endpoint.
+Endpoint for S3 API.
+Required when using an S3 clone.
+Enter a value. Press Enter to leave empty.
+endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+...
+acl> 1
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+This will leave your config looking something like:
+
+```
+[r2]
+type = s3
+provider = Cloudflare
+access_key_id = ACCESS_KEY
+secret_access_key = SECRET_ACCESS_KEY
+region = auto
+endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com
+acl = private
+```
+
+Now run `rclone lsf r2:` to see your buckets and `rclone lsf
+r2:bucket` to look within a bucket.
+
 ### Dreamhost
 
 Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is
diff --git a/fs/open_options.go b/fs/open_options.go
index 8d997118a..84e43045c 100644
--- a/fs/open_options.go
+++ b/fs/open_options.go
@@ -138,8 +138,14 @@ func (o *RangeOption) Decode(size int64) (offset, limit int64) {
 // absolute fetch using the size passed in and makes sure the range does
 // not exceed filesize. Some remotes (e.g. Onedrive, Box) don't support
 // range requests which index from the end.
+//
+// It also adjusts any SeekOption~s, turning them into absolute
+// RangeOption~s instead.
 func FixRangeOption(options []OpenOption, size int64) {
-	if size == 0 {
+	if size < 0 {
+		// Can't do anything for unknown length objects
+		return
+	} else if size == 0 {
 		// if size 0 then remove RangeOption~s
 		// replacing with a NullOptions~s which won't be rendered
 		for i := range options {
@@ -150,18 +156,21 @@ func FixRangeOption(options []OpenOption, size int64) {
 		}
 		return
 	}
-	for i := range options {
-		option := options[i]
-		if x, ok := option.(*RangeOption); ok {
+	for i, option := range options {
+		switch x := option.(type) {
+		case *RangeOption:
 			// If start is < 0 then fetch from the end
 			if x.Start < 0 {
 				x = &RangeOption{Start: size - x.End, End: -1}
 				options[i] = x
 			}
-			if x.End > size {
+			// If end is too big or undefined, fetch to the end
+			if x.End > size || x.End < 0 {
 				x = &RangeOption{Start: x.Start, End: size - 1}
 				options[i] = x
 			}
+		case *SeekOption:
+			options[i] = &RangeOption{Start: x.Offset, End: size - 1}
 		}
 	}
 }
diff --git a/lib/readers/fakeseeker.go b/lib/readers/fakeseeker.go
new file mode 100644
index 000000000..844fc0422
--- /dev/null
+++ b/lib/readers/fakeseeker.go
@@ -0,0 +1,72 @@
+package readers
+
+import (
+	"errors"
+	"fmt"
+	"io"
+)
+
+// FakeSeeker adapts an io.Seeker into an io.ReadSeeker
+type FakeSeeker struct {
+	in      io.Reader
+	readErr error
+	length  int64
+	offset  int64
+	read    bool
+}
+
+// NewFakeSeeker creates a fake io.ReadSeeker from an io.Reader
+//
+// This can be seeked before reading to discover the length passed in.
+func NewFakeSeeker(in io.Reader, length int64) io.ReadSeeker {
+	if rs, ok := in.(io.ReadSeeker); ok {
+		return rs
+	}
+	return &FakeSeeker{
+		in:     in,
+		length: length,
+	}
+}
+
+// Seek the stream - possible only before reading
+func (r *FakeSeeker) Seek(offset int64, whence int) (abs int64, err error) {
+	if r.readErr != nil {
+		return 0, r.readErr
+	}
+	if r.read {
+		return 0, fmt.Errorf("FakeSeeker: can't Seek(%d, %d) after reading", offset, whence)
+	}
+	switch whence {
+	case io.SeekStart:
+		abs = offset
+	case io.SeekCurrent:
+		abs = r.offset + offset
+	case io.SeekEnd:
+		abs = r.length + offset
+	default:
+		return 0, errors.New("FakeSeeker: invalid whence")
+	}
+	if abs < 0 {
+		return 0, errors.New("FakeSeeker: negative position")
+	}
+	r.offset = abs
+	return abs, nil
+}
+
+// Read data from the stream. Will give an error if seeked.
+func (r *FakeSeeker) Read(p []byte) (n int, err error) {
+	if r.readErr != nil {
+		return 0, r.readErr
+	}
+	if !r.read && r.offset != 0 {
+		return 0, errors.New("FakeSeeker: not at start: can't read")
+	}
+	n, err = r.in.Read(p)
+	if n != 0 {
+		r.read = true
+	}
+	if err != nil {
+		r.readErr = err
+	}
+	return n, err
+}


Test patch:
diff --git a/fs/open_options_test.go b/fs/open_options_test.go
index 52fc42ddb..829f45c19 100644
--- a/fs/open_options_test.go
+++ b/fs/open_options_test.go
@@ -149,6 +149,16 @@ func TestFixRangeOptions(t *testing.T) {
 			in:   []OpenOption{},
 			want: []OpenOption{},
 		},
+		{
+			name: "Unknown size -1",
+			in: []OpenOption{
+				&RangeOption{Start: 1, End: -1},
+			},
+			want: []OpenOption{
+				&RangeOption{Start: 1, End: -1},
+			},
+			size: -1,
+		},
 		{
 			name: "Fetch a range with size=0",
 			in: []OpenOption{
@@ -183,7 +193,7 @@ func TestFixRangeOptions(t *testing.T) {
 				&RangeOption{Start: 1, End: -1},
 			},
 			want: []OpenOption{
-				&RangeOption{Start: 1, End: -1},
+				&RangeOption{Start: 1, End: 99},
 			},
 			size: 100,
 		},
@@ -193,7 +203,7 @@ func TestFixRangeOptions(t *testing.T) {
 				&RangeOption{Start: -1, End: 10},
 			},
 			want: []OpenOption{
-				&RangeOption{Start: 90, End: -1},
+				&RangeOption{Start: 90, End: 99},
 			},
 			size: 100,
 		},
@@ -207,6 +217,20 @@ func TestFixRangeOptions(t *testing.T) {
 			},
 			size: 100,
 		},
+		{
+			name: "SeekOption",
+			in: []OpenOption{
+				&HTTPOption{Key: "a", Value: "1"},
+				&SeekOption{Offset: 10},
+				&HTTPOption{Key: "b", Value: "2"},
+			},
+			want: []OpenOption{
+				&HTTPOption{Key: "a", Value: "1"},
+				&RangeOption{Start: 10, End: 99},
+				&HTTPOption{Key: "b", Value: "2"},
+			},
+			size: 100,
+		},
 	} {
 		FixRangeOption(test.in, test.size)
 		assert.Equal(t, test.want, test.in, test.name)
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index ca36aa7d8..5b4dd86e6 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -204,6 +204,11 @@ backends:
  - backend:  "s3"
    remote:   "TestS3Alibaba:"
    fastlist: true
+ - backend:  "s3"
+   remote:   "TestS3R2:"
+   fastlist: true
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/SetTier
  - backend:  "sftp"
    remote:   "TestSFTPOpenssh:"
    fastlist: false
diff --git a/lib/readers/fakeseeker_test.go b/lib/readers/fakeseeker_test.go
new file mode 100644
index 000000000..0702b7cbb
--- /dev/null
+++ b/lib/readers/fakeseeker_test.go
@@ -0,0 +1,87 @@
+package readers
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Check interface
+var _ io.ReadSeeker = &FakeSeeker{}
+
+func TestFakeSeeker(t *testing.T) {
+	// Test that passing in an io.ReadSeeker just passes it through
+	bufReader := bytes.NewReader([]byte{1})
+	r := NewFakeSeeker(bufReader, 5)
+	assert.Equal(t, r, bufReader)
+
+	in := bytes.NewBufferString("hello")
+	buf := make([]byte, 16)
+	r = NewFakeSeeker(in, 5)
+	assert.NotEqual(t, r, in)
+
+	// check the seek offset is as passed in
+	checkPos := func(pos int64) {
+		abs, err := r.Seek(0, io.SeekCurrent)
+		require.NoError(t, err)
+		assert.Equal(t, pos, abs)
+	}
+
+	// Test some seeking
+	checkPos(0)
+
+	abs, err := r.Seek(2, io.SeekStart)
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), abs)
+	checkPos(2)
+
+	abs, err = r.Seek(-1, io.SeekEnd)
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), abs)
+	checkPos(4)
+
+	// Check can't read if not at start
+	_, err = r.Read(buf)
+	require.ErrorContains(t, err, "not at start")
+
+	// Seek back to start
+	abs, err = r.Seek(-4, io.SeekCurrent)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), abs)
+	checkPos(0)
+
+	_, err = r.Seek(42, 17)
+	require.ErrorContains(t, err, "invalid whence")
+
+	_, err = r.Seek(-1, io.SeekStart)
+	require.ErrorContains(t, err, "negative position")
+
+	// Test reading now seeked back to the start
+	n, err := r.Read(buf)
+	require.NoError(t, err)
+	assert.Equal(t, 5, n)
+	assert.Equal(t, []byte("hello"), buf[:5])
+
+	// Seeking should give an error now
+	_, err = r.Seek(-1, io.SeekEnd)
+	require.ErrorContains(t, err, "after reading")
+}
+
+func TestFakeSeekerError(t *testing.T) {
+	in := bytes.NewBufferString("hello")
+	r := NewFakeSeeker(in, 5)
+	assert.NotEqual(t, r, in)
+
+	buf, err := io.ReadAll(r)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("hello"), buf)
+
+	_, err = r.Read(buf)
+	assert.Equal(t, io.EOF, err)
+
+	_, err = r.Seek(0, io.SeekStart)
+	assert.Equal(t, io.EOF, err)
+}