Repository: rclone/rclone
Pull request #6603 (2022-12-05 22:24:41)

one word typo in the docs - 'remove' should be 'remote'
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?

<!--
one word typo in the documentation.
-->

#### Was the change discussed in an issue or in the forum before?

<!--
kinda, sorta, but not really
-->

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [ ] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 9de4a7e6a..0cd82bb12 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -2730,6 +2730,7 @@ func setQuirks(opt *Options) {
 		mightGzip = false // Never auto gzips objects
 	case "Alibaba":
 		useMultipartEtag = false // Alibaba seems to calculate multipart Etags differently from AWS
+		listObjectsV2 = false    // See #6600
 	case "HuaweiOBS":
 		// Huawei OBS PFS is not support listObjectV2, and if turn on the urlEncodeListing, marker will not work and keep list same page forever.
 		urlEncodeListings = false
@@ -3052,7 +3053,8 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
 		}
 		o.setMD5FromEtag(aws.StringValue(info.ETag))
 		o.bytes = aws.Int64Value(info.Size)
-		o.storageClass = info.StorageClass
+		storageClass := *info.StorageClass // To prevent reference to large XML structures
+		o.storageClass = &storageClass
 		o.versionID = versionID
 	} else if !o.fs.opt.NoHeadObject {
 		err := o.readMetaData(ctx) // reads info and meta, returning an error
@@ -3194,6 +3196,9 @@ func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versi
 	if err != nil {
 		return nil, nil, err
 	}
+	if aws.BoolValue(resp.IsTruncated) && (resp.ContinuationToken == nil || *resp.ContinuationToken == "") {
+		return nil, nil, errors.New("s3 protocol error: received listing v2 with IsTruncated set and no ContinuationToken. Should you be using `--s3-list-version 1`?")
+	}
 	ls.req.ContinuationToken = resp.NextContinuationToken
 	return resp, nil, nil
 }
diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go
index 4aaad2a3a..3662fc58c 100644
--- a/cmd/serve/restic/restic.go
+++ b/cmd/serve/restic/restic.go
@@ -46,7 +46,7 @@ func init() {
 var Command = &cobra.Command{
 	Use:   "restic remote:path",
 	Short: `Serve the remote for restic's REST API.`,
-	Long: `Run a basic web server to serve a remove over restic's REST backend
+	Long: `Run a basic web server to serve a remote over restic's REST backend
 API over HTTP.  This allows restic to use rclone as a data storage
 mechanism for cloud providers that restic does not support directly.
 
diff --git a/cmd/tree/tree.go b/cmd/tree/tree.go
index ae776b067..ca2999aac 100644
--- a/cmd/tree/tree.go
+++ b/cmd/tree/tree.go
@@ -61,7 +61,6 @@ func init() {
 	flags.StringVarP(cmdFlags, &sort, "sort", "", "", "Select sort: name,version,size,mtime,ctime")
 	// Graphics
 	flags.BoolVarP(cmdFlags, &opts.NoIndent, "noindent", "", false, "Don't print indentation lines")
-	flags.BoolVarP(cmdFlags, &opts.Colorize, "color", "C", false, "Turn colorization on always")
 }
 
 var commandDefinition = &cobra.Command{
@@ -116,6 +115,7 @@ For a more interactive navigation of the remote see the
 		opts.SizeSort = sort == "size"
 		ci := fs.GetConfig(context.Background())
 		opts.UnitSize = ci.HumanReadable
+		opts.Colorize = ci.TerminalColorMode != fs.TerminalColorModeNever
 		if opts.DeepLevel == 0 {
 			opts.DeepLevel = ci.MaxDepth
 		}
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 5209d1362..eb0253776 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -665,3 +665,6 @@ put them back in again.` >}}
   * Kamui <fin-kamui@pm.me>
   * asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com>
   * Nathaniel Wesley Filardo <nfilardo@microsoft.com>
+  * ycdtosa <ycdtosa@users.noreply.github.com>
+  * Erik Agterdenbos <agterdenbos@users.noreply.github.com>
+  * Kevin Verstaen <48050031+kverstae@users.noreply.github.com>
diff --git a/docs/content/commands/rclone_tree.md b/docs/content/commands/rclone_tree.md
index 1995a108f..3ecc450e5 100644
--- a/docs/content/commands/rclone_tree.md
+++ b/docs/content/commands/rclone_tree.md
@@ -48,7 +48,6 @@ rclone tree remote:path [flags]
 
 ```
   -a, --all             All files are listed (list . files too)
-  -C, --color           Turn colorization on always
   -d, --dirs-only       List directories only
       --dirsfirst       List directories before files (-U disables)
       --full-path       Print the full path prefix for each file
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 4b58920a2..1afa2a9a0 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -823,6 +823,16 @@ quicker than without the `--checksum` flag.
 When using this flag, rclone won't update mtimes of remote files if
 they are incorrect as it would normally.
 
+### --color WHEN ###
+
+Specifiy when colors (and other ANSI codes) should be added to the output.
+
+`AUTO` (default) only allows ANSI codes when the output is a terminal
+
+`NEVER` never allow ANSI codes
+
+`ALWAYS` always add ANSI codes, regardless of the output format (terminal or file)
+
 ### --compare-dest=DIR ###
 
 When using `sync`, `copy` or `move` DIR is checked in addition to the
diff --git a/docs/content/flags.md b/docs/content/flags.md
index 1cdce0e1d..c94a9e904 100644
--- a/docs/content/flags.md
+++ b/docs/content/flags.md
@@ -27,6 +27,7 @@ These flags are available for every command.
   -c, --checksum                             Skip based on checksum (if available) & size, not mod-time & size
       --client-cert string                   Client SSL certificate (PEM) for mutual TLS auth
       --client-key string                    Client SSL private key (PEM) for mutual TLS auth
+      --color                                Define when colors (and other ANSI codes) should be shown AUTO|ALWAYS|NEVER (default AUTO)
       --compare-dest stringArray             Include additional comma separated server-side paths during comparison
       --config string                        Config file (default "$HOME/.config/rclone/rclone.conf")
       --contimeout duration                  Connect timeout (default 1m0s)
diff --git a/fs/config.go b/fs/config.go
index 13d523558..d496b4d88 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -142,6 +142,7 @@ type ConfigInfo struct {
 	DisableHTTPKeepAlives   bool
 	Metadata                bool
 	ServerSideAcrossConfigs bool
+	TerminalColorMode       TerminalColorMode
 }
 
 // NewConfig creates a new config with everything set to the default
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index 42b2b4d05..abd5209d7 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -142,6 +142,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.")
 	flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects")
 	flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs")
+	flags.FVarP(flagSet, &ci.TerminalColorMode, "color", "", "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS")
 }
 
 // ParseHeaders converts the strings passed in via the header flags into HTTPOptions
diff --git a/fs/terminalcolormode.go b/fs/terminalcolormode.go
new file mode 100644
index 000000000..7da58e13f
--- /dev/null
+++ b/fs/terminalcolormode.go
@@ -0,0 +1,57 @@
+package fs
+
+import (
+	"fmt"
+	"strings"
+)
+
+// TerminalColorMode describes how ANSI codes should be handled
+type TerminalColorMode byte
+
+// TerminalColorMode constants
+const (
+	TerminalColorModeAuto TerminalColorMode = iota
+	TerminalColorModeNever
+	TerminalColorModeAlways
+)
+
+var terminalColorModeToString = []string{
+	TerminalColorModeAuto:   "AUTO",
+	TerminalColorModeNever:  "NEVER",
+	TerminalColorModeAlways: "ALWAYS",
+}
+
+// String converts a TerminalColorMode to a string
+func (m TerminalColorMode) String() string {
+	if m >= TerminalColorMode(len(terminalColorModeToString)) {
+		return fmt.Sprintf("TerminalColorMode(%d)", m)
+	}
+	return terminalColorModeToString[m]
+}
+
+// Set a TerminalColorMode
+func (m *TerminalColorMode) Set(s string) error {
+	for n, name := range terminalColorModeToString {
+		if s != "" && name == strings.ToUpper(s) {
+			*m = TerminalColorMode(n)
+			return nil
+		}
+	}
+	return fmt.Errorf("unknown terminal color mode %q", s)
+}
+
+// Type of TerminalColorMode
+func (m TerminalColorMode) Type() string {
+	return "string"
+}
+
+// UnmarshalJSON converts a string/integer in JSON to a TerminalColorMode
+func (m *TerminalColorMode) UnmarshalJSON(in []byte) error {
+	return UnmarshalJSONFlag(in, m, func(i int64) error {
+		if i < 0 || i >= int64(len(terminalColorModeToString)) {
+			return fmt.Errorf("out of range terminal color mode %d", i)
+		}
+		*m = (TerminalColorMode)(i)
+		return nil
+	})
+}
diff --git a/lib/terminal/terminal.go b/lib/terminal/terminal.go
index 4eda1abab..532cf1fcc 100644
--- a/lib/terminal/terminal.go
+++ b/lib/terminal/terminal.go
@@ -3,12 +3,14 @@
 package terminal
 
 import (
+	"context"
 	"io"
 	"os"
 	"runtime"
 	"sync"
 
 	colorable "github.com/mattn/go-colorable"
+	"github.com/rclone/rclone/fs"
 )
 
 // VT100 codes
@@ -73,13 +75,21 @@ var (
 // Start the terminal - must be called before use
 func Start() {
 	once.Do(func() {
+		ci := fs.GetConfig(context.Background())
+
 		f := os.Stdout
 		if !IsTerminal(int(f.Fd())) {
-			// If stdout not a tty then remove escape codes
-			Out = colorable.NewNonColorable(f)
+			// If stdout is not a tty, remove escape codes EXCEPT if terminal color mode equals "ALWAYS"
+			if ci.TerminalColorMode == fs.TerminalColorModeAlways {
+				Out = colorable.NewColorable(f)
+			} else {
+				Out = colorable.NewNonColorable(f)
+			}
 		} else if runtime.GOOS == "windows" && os.Getenv("TERM") != "" {
 			// If TERM is set just use stdout
 			Out = f
+		} else if ci.TerminalColorMode == fs.TerminalColorModeNever {
+			Out = colorable.NewNonColorable(f)
 		} else {
 			Out = colorable.NewColorable(f)
 		}


Test patch:
diff --git a/fs/terminalcolormode_test.go b/fs/terminalcolormode_test.go
new file mode 100644
index 000000000..6060b4342
--- /dev/null
+++ b/fs/terminalcolormode_test.go
@@ -0,0 +1,74 @@
+package fs
+
+import (
+	"encoding/json"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTerminalColorModeString(t *testing.T) {
+	for _, test := range []struct {
+		in   TerminalColorMode
+		want string
+	}{
+		{TerminalColorModeAuto, "AUTO"},
+		{TerminalColorModeAlways, "ALWAYS"},
+		{TerminalColorModeNever, "NEVER"},
+		{36, "TerminalColorMode(36)"},
+	} {
+		tcm := test.in
+		assert.Equal(t, test.want, tcm.String(), test.in)
+	}
+}
+
+func TestTerminalColorModeSet(t *testing.T) {
+	for _, test := range []struct {
+		in          string
+		want        TerminalColorMode
+		expectError bool
+	}{
+		{"auto", TerminalColorModeAuto, false},
+		{"ALWAYS", TerminalColorModeAlways, false},
+		{"Never", TerminalColorModeNever, false},
+		{"INVALID", 0, true},
+	} {
+		tcm := TerminalColorMode(0)
+		err := tcm.Set(test.in)
+		if test.expectError {
+			require.Error(t, err, test.in)
+		} else {
+			require.NoError(t, err, test.in)
+		}
+		assert.Equal(t, test.want, tcm, test.in)
+	}
+}
+
+func TestTerminalColorModeUnmarshalJSON(t *testing.T) {
+	for _, test := range []struct {
+		in          string
+		want        TerminalColorMode
+		expectError bool
+	}{
+		{`"auto"`, TerminalColorModeAuto, false},
+		{`"ALWAYS"`, TerminalColorModeAlways, false},
+		{`"Never"`, TerminalColorModeNever, false},
+		{`"Invalid"`, 0, true},
+		{strconv.Itoa(int(TerminalColorModeAuto)), TerminalColorModeAuto, false},
+		{strconv.Itoa(int(TerminalColorModeAlways)), TerminalColorModeAlways, false},
+		{strconv.Itoa(int(TerminalColorModeNever)), TerminalColorModeNever, false},
+		{`99`, 0, true},
+		{`-99`, 0, true},
+	} {
+		var tcm TerminalColorMode
+		err := json.Unmarshal([]byte(test.in), &tcm)
+		if test.expectError {
+			require.Error(t, err, test.in)
+		} else {
+			require.NoError(t, err, test.in)
+		}
+		assert.Equal(t, test.want, tcm, test.in)
+	}
+}