Repository: rclone/rclone
Pull request #6101 (2022-04-12 19:28:40)

googlecloudstorage: add --gcs-no-check-bucket to minimise transactions and perms
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?

Adds a configuration option to the GCS backend to allow skipping the
check if a bucket exists before copying an object to it, much like
f406dbb added for S3.

#### Was the change discussed in an issue or in the forum before?

This is essentially a GCS-y version of the fix for #4449

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go
index 62c30e4a2..9a121349d 100644
--- a/backend/googlecloudstorage/googlecloudstorage.go
+++ b/backend/googlecloudstorage/googlecloudstorage.go
@@ -295,6 +295,15 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 				Value: "DURABLE_REDUCED_AVAILABILITY",
 				Help:  "Durable reduced availability storage class",
 			}},
+		}, {
+			Name: "no_check_bucket",
+			Help: `If set, don't attempt to check the bucket exists or create it.
+
+This can be useful when trying to minimise the number of transactions
+rclone does if you know the bucket exists already.
+`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -317,6 +326,7 @@ type Options struct {
 	BucketPolicyOnly          bool                 `config:"bucket_policy_only"`
 	Location                  string               `config:"location"`
 	StorageClass              string               `config:"storage_class"`
+	NoCheckBucket             bool                 `config:"no_check_bucket"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -840,6 +850,14 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 	}, nil)
 }
 
+// checkBucket creates the bucket if it doesn't exist unless NoCheckBucket is true
+func (f *Fs) checkBucket(ctx context.Context, bucket string) error {
+	if f.opt.NoCheckBucket {
+		return nil
+	}
+	return f.makeBucket(ctx, bucket)
+}
+
 // Rmdir deletes the bucket if the fs is at the root
 //
 // Returns an error if it isn't empty: Error 409: The bucket you tried
@@ -873,7 +891,7 @@ func (f *Fs) Precision() time.Duration {
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	dstBucket, dstPath := f.split(remote)
-	err := f.makeBucket(ctx, dstBucket)
+	err := f.checkBucket(ctx, dstBucket)
 	if err != nil {
 		return nil, err
 	}
@@ -1123,7 +1141,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	bucket, bucketPath := o.split()
-	err := o.fs.makeBucket(ctx, bucket)
+	err := o.fs.checkBucket(ctx, bucket)
 	if err != nil {
 		return err
 	}
diff --git a/bin/build-xgo-cgofuse.sh b/bin/build-xgo-cgofuse.sh
index 8e4c0259e..01908dfcc 100755
--- a/bin/build-xgo-cgofuse.sh
+++ b/bin/build-xgo-cgofuse.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 set -e
-docker build -t rclone/xgo-cgofuse https://github.com/billziss-gh/cgofuse.git
+docker build -t rclone/xgo-cgofuse https://github.com/winfsp/cgofuse.git
 docker images
 docker push rclone/xgo-cgofuse
diff --git a/cmd/cmount/fs.go b/cmd/cmount/fs.go
index d627a9829..9e034da55 100644
--- a/cmd/cmount/fs.go
+++ b/cmd/cmount/fs.go
@@ -13,7 +13,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/billziss-gh/cgofuse/fuse"
+	"github.com/winfsp/cgofuse/fuse"
 	"github.com/rclone/rclone/cmd/mountlib"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
diff --git a/cmd/cmount/mount.go b/cmd/cmount/mount.go
index cafecafac..df92b792b 100644
--- a/cmd/cmount/mount.go
+++ b/cmd/cmount/mount.go
@@ -18,7 +18,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/billziss-gh/cgofuse/fuse"
+	"github.com/winfsp/cgofuse/fuse"
 	"github.com/rclone/rclone/cmd/mountlib"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/lib/atexit"
diff --git a/cmd/mountlib/help.go b/cmd/mountlib/help.go
index 641097c9d..10aea7b9f 100644
--- a/cmd/mountlib/help.go
+++ b/cmd/mountlib/help.go
@@ -65,10 +65,10 @@ at all, then 1 PiB is set as both the total and the free size.
 To run rclone @ on Windows, you will need to
 download and install [WinFsp](http://www.secfs.net/winfsp/).
 
-[WinFsp](https://github.com/billziss-gh/winfsp) is an open-source
+[WinFsp](https://github.com/winfsp/winfsp) is an open-source
 Windows File System Proxy which makes it easy to write user space file
 systems for Windows.  It provides a FUSE emulation layer which rclone
-uses combination with [cgofuse](https://github.com/billziss-gh/cgofuse).
+uses combination with [cgofuse](https://github.com/winfsp/cgofuse).
 Both of these packages are by Bill Zissimopoulos who was very helpful
 during the implementation of rclone @ for Windows.
 
@@ -218,7 +218,7 @@ from Microsoft's Sysinternals suite, which has option |-s| to start
 processes as the SYSTEM account. Another alternative is to run the mount
 command from a Windows Scheduled Task, or a Windows Service, configured
 to run as the SYSTEM account. A third alternative is to use the
-[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)).
+[WinFsp.Launcher infrastructure](https://github.com/winfsp/winfsp/wiki/WinFsp-Service-Architecture)).
 Note that when running rclone as another user, it will not use
 the configuration file from your profile unless you tell it to
 with the [|--config|](https://rclone.org/docs/#config-config-file) option.
diff --git a/docs/content/netstorage.md b/docs/content/netstorage.md
index 22587dbed..428fcc361 100644
--- a/docs/content/netstorage.md
+++ b/docs/content/netstorage.md
@@ -277,4 +277,10 @@ the object that will be the target of the symlink (for example, /links/mylink).
 Include the file extension for the object, if applicable.
 `rclone backend symlink <src> <path>`
 
+## Support
+
+If you have any questions or issues, please contact [Akamai Technical Support
+via Control Center or by
+phone](https://control.akamai.com/apps/support-ui/#/contact-support).
+
 {{< rem autogenerated options stop >}}
diff --git a/go.mod b/go.mod
index aa3f59883..cf8d5c41b 100644
--- a/go.mod
+++ b/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/artyom/mtab v1.0.0
 	github.com/atotto/clipboard v0.1.4
 	github.com/aws/aws-sdk-go v1.43.30
-	github.com/billziss-gh/cgofuse v1.5.0
+	github.com/winfsp/cgofuse v1.5.0
 	github.com/buengese/sgzip v0.1.1
 	github.com/colinmarc/hdfs/v2 v2.3.0
 	github.com/coreos/go-semver v0.3.0
diff --git a/go.sum b/go.sum
index d1e635ad1..4ebd7f6bf 100644
--- a/go.sum
+++ b/go.sum
@@ -591,6 +591,8 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI
 github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
 github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/winfsp/cgofuse v1.5.0 h1:MsBP7Mi/LiJf/7/F3O/7HjjR009ds6KCdqXzKpZSWxI=
+github.com/winfsp/cgofuse v1.5.0/go.mod h1:h3awhoUOcn2VYVKCwDaYxSLlZwnyK+A8KaDoLUp2lbU=
 github.com/xanzy/ssh-agent v0.3.1 h1:AmzO1SSWxw73zxFZPRwaMN1MohDw8UyHnmuxyceTEGo=
 github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=


Test patch:
diff --git a/cmd/test/makefiles/makefiles.go b/cmd/test/makefiles/makefiles.go
index 26263e56a..83a1f5842 100644
--- a/cmd/test/makefiles/makefiles.go
+++ b/cmd/test/makefiles/makefiles.go
@@ -36,6 +36,7 @@ var (
 	sparse                   = false
 	ascii                    = false
 	pattern                  = false
+	chargen                  = false
 
 	// Globals
 	randSource          *rand.Rand
@@ -65,7 +66,8 @@ func init() {
 		flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00")
 		flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)")
 		flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only")
-		flags.BoolVarP(f, &pattern, "pattern", "", ascii, "Fill files with a periodic pattern")
+		flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern")
+		flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern")
 	}
 }
 
@@ -139,8 +141,8 @@ func commonInit() {
 		fs.Logf(nil, "Using random seed = %d", seed)
 	}
 	randSource = rand.New(rand.NewSource(seed))
-	if bool2int(zero)+bool2int(sparse)+bool2int(ascii)+bool2int(pattern) > 1 {
-		log.Fatal("Can only supply one of --zero, --sparse, --ascii or --pattern")
+	if bool2int(zero)+bool2int(sparse)+bool2int(ascii)+bool2int(pattern)+bool2int(chargen) > 1 {
+		log.Fatal("Can only supply one of --zero, --sparse, --ascii, --pattern or --chargen")
 	}
 	switch {
 	case zero, sparse:
@@ -149,6 +151,8 @@ func commonInit() {
 		source = asciiReader{}
 	case pattern:
 		source = readers.NewPatternReader(math.MaxInt64)
+	case chargen:
+		source = &chargenReader{}
 	default:
 		source = randSource
 	}
@@ -178,6 +182,38 @@ func (asciiReader) Read(p []byte) (n int, err error) {
 	return n, err
 }
 
+type chargenReader struct {
+	start   byte // offset from startChar to start line with
+	written byte // chars in line so far
+}
+
+// Read a chunk of printable ASCII characters in chargen format
+func (r *chargenReader) Read(p []byte) (n int, err error) {
+	const (
+		startChar    = 0x20 // ' '
+		endChar      = 0x7E // '~' inclusive
+		charsPerLine = 72
+	)
+	for i := range p {
+		if r.written >= charsPerLine {
+			r.start++
+			if r.start > endChar-startChar {
+				r.start = 0
+			}
+			p[i] = '\n'
+			r.written = 0
+		} else {
+			c := r.start + r.written + startChar
+			if c > endChar {
+				c -= endChar - startChar + 1
+			}
+			p[i] = c
+			r.written++
+		}
+	}
+	return len(p), err
+}
+
 // fileName creates a unique random file or directory name
 func fileName() (name string) {
 	for {