Repository: rclone/rclone
Pull request #5030 (2021-02-16 18:44:47)

Fix shared_credentials_file s3 auth
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?
Fix shared_credentials_file s3 auth option
<!--
Describe the changes here
-->

#### Was the change discussed in an issue or in the forum before?
https://forum.rclone.org/t/shared-credentials-file-config-option-not-works/22318/6
<!--
Link issues and relevant forum posts here.
-->

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [ ] I have added tests for all changes in this PR if appropriate.
- [ ] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d8f4abebb..98adc0295 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -24,7 +24,7 @@ jobs:
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -34,7 +34,7 @@ jobs:
 
           - job_name: mac_amd64
             os: macOS-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -43,14 +43,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macOS-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -macos-sdk macosx11.1 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows_amd64
             os: windows-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             build_flags: '-include "^windows/amd64" -cgo'
             build_args: '-buildmode exe'
@@ -60,7 +60,7 @@ jobs:
 
           - job_name: windows_386
             os: windows-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             gotags: cmount
             goarch: '386'
             cgo: '1'
@@ -71,7 +71,7 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.16.0-rc1'
+            go: '1.16.x'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go
index f3794e745..aa07b7a68 100644
--- a/backend/crypt/crypt.go
+++ b/backend/crypt/crypt.go
@@ -101,6 +101,21 @@ names, or for debugging purposes.`,
 			Default:  false,
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
+		}, {
+			Name:     "no_data_encryption",
+			Help:     "Option to either encrypt file data or leave it unencrypted.",
+			Default:  false,
+			Advanced: true,
+			Examples: []fs.OptionExample{
+				{
+					Value: "true",
+					Help:  "Don't encrypt file data, leave it unencrypted.",
+				},
+				{
+					Value: "false",
+					Help:  "Encrypt file data.",
+				},
+			},
 		}},
 	})
 }
@@ -209,6 +224,7 @@ type Options struct {
 	Remote                  string `config:"remote"`
 	FilenameEncryption      string `config:"filename_encryption"`
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
+	NoDataEncryption        bool   `config:"no_data_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
@@ -346,6 +362,10 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
+	if f.opt.NoDataEncryption {
+		return put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
+	}
+
 	// Encrypt the data into wrappedIn
 	wrappedIn, encrypter, err := f.cipher.encryptData(in)
 	if err != nil {
@@ -617,6 +637,10 @@ func (f *Fs) computeHashWithNonce(ctx context.Context, nonce nonce, src fs.Objec
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
+	if f.opt.NoDataEncryption {
+		return src.Hash(ctx, hashType)
+	}
+
 	// Read the nonce - opening the file is sufficient to read the nonce in
 	// use a limited read so we only read the header
 	in, err := o.Object.Open(ctx, &fs.RangeOption{Start: 0, End: int64(fileHeaderSize) - 1})
@@ -822,9 +846,13 @@ func (o *Object) Remote() string {
 
 // Size returns the size of the file
 func (o *Object) Size() int64 {
-	size, err := o.f.cipher.DecryptedSize(o.Object.Size())
-	if err != nil {
-		fs.Debugf(o, "Bad size for decrypt: %v", err)
+	size := o.Object.Size()
+	if !o.f.opt.NoDataEncryption {
+		var err error
+		size, err = o.f.cipher.DecryptedSize(size)
+		if err != nil {
+			fs.Debugf(o, "Bad size for decrypt: %v", err)
+		}
 	}
 	return size
 }
@@ -842,6 +870,10 @@ func (o *Object) UnWrap() fs.Object {
 
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
+	if o.f.opt.NoDataEncryption {
+		return o.Object.Open(ctx, options...)
+	}
+
 	var openOptions []fs.OpenOption
 	var offset, limit int64 = 0, -1
 	for _, option := range options {
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index ede7e2a8d..e230fc6d8 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -1559,9 +1559,6 @@ func s3Connection(ctx context.Context, opt *Options) (*s3.S3, *session.Session,
 	if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" {
 		// Enable loading config options from ~/.aws/config (selected by AWS_PROFILE env)
 		awsSessionOpts.SharedConfigState = session.SharedConfigEnable
-		// The session constructor (aws/session/mergeConfigSrcs) will only use the user's preferred credential source
-		// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
-		awsSessionOpts.Config.Credentials = nil
 	}
 	ses, err := session.NewSessionWithOptions(awsSessionOpts)
 	if err != nil {
diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go
index d6b01cf2e..036505b7b 100644
--- a/cmd/mountlib/mount.go
+++ b/cmd/mountlib/mount.go
@@ -321,10 +321,24 @@ Note that the mapping of permissions is not always trivial, and the result
 you see in Windows Explorer may not be exactly like you expected.
 For example, when setting a value that includes write access, this will be
 mapped to individual permissions "write attributes", "write data" and "append data",
-but not "write extended attributes" (WinFsp does not support extended attributes,
-see [this](https://github.com/billziss-gh/winfsp/wiki/NTFS-Compatibility)).
-Windows will then show this as basic permission "Special" instead of "Write",
-because "Write" includes the "write extended attributes" permission.
+but not "write extended attributes". Windows will then show this as basic
+permission "Special" instead of "Write", because "Write" includes the
+"write extended attributes" permission.
+
+If you set POSIX permissions for only allowing access to the owner, using
+|--file-perms 0600 --dir-perms 0700|, the user group and the built-in "Everyone"
+group will still be given some special permissions, such as "read attributes"
+and "read permissions", in Windows. This is done for compatibility reasons,
+e.g. to allow users without additional permissions to be able to read basic
+metadata about files like in UNIX. One case that may arise is that other programs
+(incorrectly) interprets this as the file being accessible by everyone. For example
+an SSH client may warn about "unprotected private key file".
+
+WinFsp 2021 (version 1.9, still in beta) introduces a new FUSE option "FileSecurity",
+that allows the complete specification of file security descriptors using
+[SDDL](https://docs.microsoft.com/en-us/windows/win32/secauthz/security-descriptor-string-format).
+With this you can work around issues such as the mentioned "unprotected private key file"
+by specifying |-o FileSecurity="D:P(A;;FA;;;OW)"|, for file all access (FA) to the owner (OW).
 
 #### Windows caveats
 
diff --git a/docs/content/cache.md b/docs/content/cache.md
index c2d6f7de9..c18ab19e1 100644
--- a/docs/content/cache.md
+++ b/docs/content/cache.md
@@ -205,7 +205,7 @@ These URLs are used by Plex internally to connect to the Plex server securely.
 
 The format for these URLs is the following:
 
-https://ip-with-dots-replaced.server-hash.plex.direct:32400/
+`https://ip-with-dots-replaced.server-hash.plex.direct:32400/`
 
 The `ip-with-dots-replaced` part can be any IPv4 address, where the dots
 have been replaced with dashes, e.g. `127.0.0.1` becomes `127-0-0-1`.
diff --git a/docs/content/filtering.md b/docs/content/filtering.md
index 4d2bd0250..4b81694c4 100644
--- a/docs/content/filtering.md
+++ b/docs/content/filtering.md
@@ -661,7 +661,7 @@ not list `dir3`, `file3` or `.ignore`.
 ## Common pitfalls
 
 The most frequent filter support issues on
-the [rclone forum](https://https://forum.rclone.org/) are:
+the [rclone forum](https://forum.rclone.org/) are:
 
 * Not using paths relative to the root of the remote
 * Not using `/` to match from the root of a remote
diff --git a/docs/content/sftp.md b/docs/content/sftp.md
index 172220011..eb1427362 100644
--- a/docs/content/sftp.md
+++ b/docs/content/sftp.md
@@ -526,8 +526,8 @@ The Go SSH library disables the use of the aes128-cbc cipher by
 default, due to security concerns. This can be re-enabled on a
 per-connection basis by setting the `use_insecure_cipher` setting in
 the configuration file to `true`. Further details on the insecurity of
-this cipher can be found [in this paper]
-(http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf).
+this cipher can be found
+[in this paper](http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf).
 
 SFTP isn't supported under plan9 until [this
 issue](https://github.com/pkg/sftp/issues/156) is fixed.


Test patch:
diff --git a/backend/crypt/crypt_test.go b/backend/crypt/crypt_test.go
index 144c7824f..24eaa2722 100644
--- a/backend/crypt/crypt_test.go
+++ b/backend/crypt/crypt_test.go
@@ -91,3 +91,26 @@ func TestObfuscate(t *testing.T) {
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
+
+// TestNoDataObfuscate runs integration tests against the remote
+func TestNoDataObfuscate(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
+	name := "TestCrypt4"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
+			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
+			{Name: name, Key: "no_data_encryption", Value: "true"},
+		},
+		SkipBadWindowsCharacters:     true,
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}
diff --git a/fstest/testserver/init.d/docker.bash b/fstest/testserver/init.d/docker.bash
index 2bd00643d..442738986 100644
--- a/fstest/testserver/init.d/docker.bash
+++ b/fstest/testserver/init.d/docker.bash
@@ -2,13 +2,13 @@
 
 stop() {
     if status ; then
-        docker stop $NAME
+        docker stop "$NAME"
         echo "$NAME stopped"
     fi
 }
 
 status() {
-    if docker ps --format "{{.Names}}" | grep ^${NAME}$ >/dev/null ; then
+    if docker ps --format '{{.Names}}' | grep -q "^${NAME}$" ; then
         echo "$NAME running"
     else
         echo "$NAME not running"
@@ -18,5 +18,5 @@ status() {
 }
 
 docker_ip() {
-    docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $NAME    
+    docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{"\n"}}{{end}}' "$NAME" | head -1
 }
diff --git a/fstest/testserver/testserver.go b/fstest/testserver/testserver.go
index 4ec5e0acb..8026b6d6e 100644
--- a/fstest/testserver/testserver.go
+++ b/fstest/testserver/testserver.go
@@ -91,7 +91,7 @@ func start(name string) error {
 				continue
 			}
 
-			// fs.Debugf(name, "key = %q, envKey = %q, value = %q", key, envKey, value)
+			// fs.Debugf(name, "key = %q, envKey = %q, value = %q", key, envKey(name, string(key)), value)
 			err = os.Setenv(envKey(name, string(key)), string(value))
 			if err != nil {
 				return err