Repository: rclone/rclone
File: backend/s3/s3.go (Go)

Code before:
	if f.opt.RequesterPays {
		req.RequestPayer = aws.String(s3.RequestPayerRequester)
	}
	if f.opt.ServerSideEncryption != "" {
		req.ServerSideEncryption = &f.opt.ServerSideEncryption
	}
	if f.opt.SSECustomerAlgorithm != "" {
		req.SSECustomerAlgorithm = &f.opt.SSECustomerAlgorithm
		req.CopySourceSSECustomerAlgorithm = &f.opt.SSECustomerAlgorithm
	}
	if f.opt.SSECustomerKey != "" {
		req.SSECustomerKey = &f.opt.SSECustomerKey
		req.CopySourceSSECustomerKey = &f.opt.SSECustomerKey
	}
	if f.opt.SSECustomerKeyMD5 != "" {
		req.SSECustomerKeyMD5 = &f.opt.SSECustomerKeyMD5
		req.CopySourceSSECustomerKeyMD5 = &f.opt.SSECustomerKeyMD5
	}
	if f.opt.SSEKMSKeyID != "" {
		req.SSEKMSKeyId = &f.opt.SSEKMSKeyID
	}
	if req.StorageClass == nil && f.opt.StorageClass != "" {
		req.StorageClass = &f.opt.StorageClass
	}

	if src.bytes >= int64(f.opt.CopyCutoff) && f.opt.CopyCutoff > 0 {
		return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, src)
	}
	return f.pacer.Call(func() (bool, error) {
		_, err := f.c.CopyObjectWithContext(ctx, req)
		return f.shouldRetry(ctx, err)
	})
}

func calculateRange(partSize, partIndex, numParts, totalSize int64) string {
	start := partIndex * partSize
	var ends string
	if partIndex == numParts-1 {
		if totalSize >= 1 {
			ends = strconv.FormatInt(totalSize-1, 10)
		}
	} else {
		ends = strconv.FormatInt(start+partSize-1, 10)
	}
	return fmt.Sprintf("bytes=%v-%v", start, ends)
}

func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, src *Object) (err error) {
	info, err := src.headObject(ctx)
	if err != nil {
		return err

Review comment (bug):
This works fine unless the size of the object is > 5 GB then the single part copy will fail.

I think we should probably insert a check here instead and return an error to the user when initialising the Fs

https://github.com/rclone/rclone/blob/52d444f4a91cc7afd1552574dced484fd3b5138b/backend/s3/s3.go#L3252

Code after:
// makeBucket creates the bucket if it doesn't exist
func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
	if f.opt.NoCheckBucket {
		return nil
	}
	return f.cache.Create(bucket, func() error {
		req := s3.CreateBucketInput{
			Bucket: &bucket,
			ACL:    stringPointerOrNil(f.opt.BucketACL),
		}
		if f.opt.LocationConstraint != "" {
			req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
				LocationConstraint: &f.opt.LocationConstraint,
			}
		}
		err := f.pacer.Call(func() (bool, error) {
			_, err := f.c.CreateBucketWithContext(ctx, &req)
			return f.shouldRetry(ctx, err)
		})
		if err == nil {
			fs.Infof(f, "Bucket %q created with ACL %q", bucket, f.opt.BucketACL)
		}
		if awsErr, ok := err.(awserr.Error); ok {
			switch awsErr.Code() {
			case "BucketAlreadyOwnedByYou":
				err = nil
			case "BucketAlreadyExists", "BucketNameUnavailable":
				if f.opt.UseAlreadyExists.Value {
					// We can trust BucketAlreadyExists to mean not owned by us, so make it non retriable
					err = fserrors.NoRetryError(err)
				} else {
					// We can't trust BucketAlreadyExists to mean not owned by us, so ignore it
					err = nil
				}
			}
		}
		return err
	}, func() (bool, error) {
		return f.bucketExists(ctx, bucket)
	})
}

// Rmdir deletes the bucket if the fs is at the root
//
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
	bucket, directory := f.split(dir)
	// Remove directory marker file
	if f.opt.DirectoryMarkers && bucket != "" && dir != "" {
		o := &Object{
			fs:     f,