Repository: rclone/rclone (language: go)

Issue:
Windows: Destination Encoding characters are improperly escaped
Hi Nick,

It feels like I'm the only person here using rclone to do local Windows → Windows transfers so here's another bug. 

Basically, the Full Width Pipe character seems to be escaped with your backtick **sometimes** resulting in very odd behavior. 

I can't fully describe it, but here's the batch files that makes things easy for you to debug. 


**control.bat** - This file contains expected behavior where `｜.txt` is successfully copied from folder `a` to folder `b`. 
```
@echo off
chcp 65001

mkdir a
mkdir b
echo "hello world" > a/｜.txt
rclone --config="" copy a b
dir b
pause
```

**bug.bat** - Here we expect `｜.txt` to be copied from folder `c` to folder `｜` but it ends up in folder `‛｜` for some reason. Note that despite the folder path ending up escaped, the text file `｜.txt` is NOT escaped. 
```
@echo off
chcp 65001

mkdir c
mkdir ｜
echo "hello world" > c/｜.txt
rclone --config="" copy c ｜
dir ｜
pause
```
Specifically, it causes statements like `rclone copy <somewhere> .` to spontaneously miss if `.` expands to a path with a Full Width replacement character.


Discussion / hints:

Fix patch:
diff --git a/backend/local/local.go b/backend/local/local.go
index 80f44c76eb285..e292e689ac5d4 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -1568,32 +1568,47 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
 }
 
 func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
-	if runtime.GOOS != "windows" || !strings.HasPrefix(s, "\\") {
-		if !filepath.IsAbs(s) {
-			s2, err := filepath.Abs(s)
-			if err == nil {
-				s = s2
-			}
-		} else {
-			s = filepath.Clean(s)
-		}
-	}
+	var vol string
 	if runtime.GOOS == "windows" {
-		s = filepath.ToSlash(s)
-		vol := filepath.VolumeName(s)
+		vol = filepath.VolumeName(s)
 		if vol == `\\?` && len(s) >= 6 {
 			// `\\?\C:`
 			vol = s[:6]
 		}
-		s = vol + enc.FromStandardPath(s[len(vol):])
-		s = filepath.FromSlash(s)
-		if !noUNC {
-			// Convert to UNC
-			s = file.UNCPath(s)
+		s = s[len(vol):]
+	}
+	// Don't use FromStandardPath. Make sure Dot (`.`, `..`) as name will not be reencoded
+	// Take care of the case Standard: ./．/‛． (the first dot means current directory)
+	if enc != encoder.Standard {
+		s = filepath.ToSlash(s)
+		parts := strings.Split(s, "/")
+		encoded := make([]string, len(parts))
+		changed := false
+		for i, p := range parts {
+			if (p == ".") || (p == "..") {
+				encoded[i] = p
+				continue
+			}
+			part := enc.FromStandardName(p)
+			changed = changed || part != p
+			encoded[i] = part
+		}
+		if changed {
+			s = strings.Join(encoded, "/")
 		}
-		return s
+		s = filepath.FromSlash(s)
+	}
+	if runtime.GOOS == "windows" {
+		s = vol + s
+	}
+	s2, err := filepath.Abs(s)
+	if err == nil {
+		s = s2
+	}
+	if !noUNC {
+		// Convert to UNC. It does nothing on non windows platforms.
+		s = file.UNCPath(s)
 	}
-	s = enc.FromStandardPath(s)
 	return s
 }
 
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 36ff231f2a28f..2b4ecd1119b2b 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -348,8 +348,8 @@ have a Windows file system with Unicode fullwidth characters
 remote rather than being translated to regular (halfwidth) `*`, `?` and `:`.
 
 The `--backend-encoding` flags allow you to change that. You can
-disable the encoding completely with `--backend-encoding None` or set
-`encoding = None` in the config file.
+disable the encoding completely with `--backend-encoding Raw` or set
+`encoding = Raw` in the config file.
 
 Encoding takes a comma separated list of encodings. You can see the
 list of all possible values by passing an invalid value to this
@@ -375,7 +375,7 @@ will show you the defaults for the backends.
 | LeftSpace | SPACE on the left of a string | `␠` |
 | LeftTilde | `~` on the left of a string | `～` |
 | LtGt | `<`, `>` | `＜`, `＞` |
-| None | No characters are encoded | |
+| None ¹ | NUL 0x00 | ␀ |
 | Percent | `%` | `％` |
 | Pipe | \| | `｜` |
 | Question | `?` | `？` |
@@ -387,6 +387,10 @@ will show you the defaults for the backends.
 | Slash | `/` | `／` |
 | SquareBracket | `[`, `]` | `［`, `］` |
 
+¹ Encoding from NUL 0x00 to ␀ is always implicit except when using Raw.
+It was previously incorrectly documented as disabling encoding,
+and to maintain backward compatibility, its behavior has not been changed.
+
 ##### Encoding example: FTP
 
 To take a specific example, the FTP backend's default encoding is
@@ -430,7 +434,7 @@ the default value but without `Colon,Question,Asterisk`:
 --local-encoding "Slash,LtGt,DoubleQuote,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot"
 ```
 
-Alternatively, you can disable the conversion of any characters with `--local-encoding None`.
+Alternatively, you can disable the conversion of any characters with `--local-encoding Raw`.
 
 Instead of using command-line argument `--local-encoding`, you may also set it
 as [environment variable](/docs/#environment-variables) `RCLONE_LOCAL_ENCODING`,
diff --git a/lib/encoder/encoder.go b/lib/encoder/encoder.go
index a14d7c8485b42..264757cd896ea 100644
--- a/lib/encoder/encoder.go
+++ b/lib/encoder/encoder.go
@@ -35,34 +35,35 @@ const (
 
 // Possible flags for the MultiEncoder
 const (
-	EncodeZero          MultiEncoder = 0         // NUL(0x00)
-	EncodeSlash         MultiEncoder = 1 << iota // /
-	EncodeLtGt                                   // <>
-	EncodeDoubleQuote                            // "
-	EncodeSingleQuote                            // '
-	EncodeBackQuote                              // `
-	EncodeDollar                                 // $
-	EncodeColon                                  // :
-	EncodeQuestion                               // ?
-	EncodeAsterisk                               // *
-	EncodePipe                                   // |
-	EncodeHash                                   // #
-	EncodePercent                                // %
-	EncodeBackSlash                              // \
-	EncodeCrLf                                   // CR(0x0D), LF(0x0A)
-	EncodeDel                                    // DEL(0x7F)
-	EncodeCtl                                    // CTRL(0x01-0x1F)
-	EncodeLeftSpace                              // Leading SPACE
-	EncodeLeftPeriod                             // Leading .
-	EncodeLeftTilde                              // Leading ~
-	EncodeLeftCrLfHtVt                           // Leading CR LF HT VT
-	EncodeRightSpace                             // Trailing SPACE
-	EncodeRightPeriod                            // Trailing .
-	EncodeRightCrLfHtVt                          // Trailing CR LF HT VT
-	EncodeInvalidUtf8                            // Invalid UTF-8 bytes
-	EncodeDot                                    // . and .. names
-	EncodeSquareBracket                          // []
-	EncodeSemicolon                              // ;
+	EncodeZero          MultiEncoder = 0 // NUL(0x00)
+	EncodeRaw           MultiEncoder = 1 << (iota - 1)
+	EncodeSlash                      // /
+	EncodeLtGt                       // <>
+	EncodeDoubleQuote                // "
+	EncodeSingleQuote                // '
+	EncodeBackQuote                  // `
+	EncodeDollar                     // $
+	EncodeColon                      // :
+	EncodeQuestion                   // ?
+	EncodeAsterisk                   // *
+	EncodePipe                       // |
+	EncodeHash                       // #
+	EncodePercent                    // %
+	EncodeBackSlash                  // \
+	EncodeCrLf                       // CR(0x0D), LF(0x0A)
+	EncodeDel                        // DEL(0x7F)
+	EncodeCtl                        // CTRL(0x01-0x1F)
+	EncodeLeftSpace                  // Leading SPACE
+	EncodeLeftPeriod                 // Leading .
+	EncodeLeftTilde                  // Leading ~
+	EncodeLeftCrLfHtVt               // Leading CR LF HT VT
+	EncodeRightSpace                 // Trailing SPACE
+	EncodeRightPeriod                // Trailing .
+	EncodeRightCrLfHtVt              // Trailing CR LF HT VT
+	EncodeInvalidUtf8                // Invalid UTF-8 bytes
+	EncodeDot                        // . and .. names
+	EncodeSquareBracket              // []
+	EncodeSemicolon                  // ;
 
 	// Synthetic
 	EncodeWin         = EncodeColon | EncodeQuestion | EncodeDoubleQuote | EncodeAsterisk | EncodeLtGt | EncodePipe // :?"*<>|
@@ -117,6 +118,7 @@ func alias(name string, mask MultiEncoder) {
 }
 
 func init() {
+	alias("Raw", EncodeRaw)
 	alias("None", EncodeZero)
 	alias("Slash", EncodeSlash)
 	alias("LtGt", EncodeLtGt)
@@ -214,6 +216,10 @@ func (mask *MultiEncoder) Scan(s fmt.ScanState, ch rune) error {
 // Encode takes a raw name and substitutes any reserved characters and
 // patterns in it
 func (mask MultiEncoder) Encode(in string) string {
+	if mask == EncodeRaw {
+		return in
+	}
+
 	if in == "" {
 		return ""
 	}
@@ -671,6 +677,10 @@ func (mask MultiEncoder) Encode(in string) string {
 
 // Decode takes a name and undoes any substitutions made by Encode
 func (mask MultiEncoder) Decode(in string) string {
+	if mask == EncodeRaw {
+		return in
+	}
+
 	if mask.Has(EncodeDot) {
 		switch in {
 		case "．":
diff --git a/lib/encoder/internal/gen/main.go b/lib/encoder/internal/gen/main.go
index 97bcb32ba0f27..6159fd9096768 100644
--- a/lib/encoder/internal/gen/main.go
+++ b/lib/encoder/internal/gen/main.go
@@ -229,8 +229,8 @@ func main() {
 			fatalW(fd.WriteString(" "))("Write:")
 		}
 		in, out := buildTestString(
-			[]mapping{getMapping(m.mask)},                               // pick
-			[]mapping{getMapping(0)},                                    // quote
+			[]mapping{getMapping(m.mask)},             // pick
+			[]mapping{getMapping(encoder.EncodeZero)}, // quote
 			printables, fullwidthPrintables, encodables, encoded, greek) // fill
 		fatalW(fmt.Fprintf(fd, `{ // %d
 		mask: %s,
@@ -262,7 +262,7 @@ var testCasesSingleEdge = []testCase{
 			for idx, orig := range e.orig {
 				replace := e.replace[idx]
 				pairs := buildEdgeTestString(
-					[]edge{e}, []mapping{getMapping(0), getMapping(m.mask)}, // quote
+					[]edge{e}, []mapping{getMapping(encoder.EncodeZero), getMapping(m.mask)}, // quote
 					[][]rune{printables, fullwidthPrintables, encodables, encoded, greek}, // fill
 					func(rIn, rOut []rune, quoteOut []bool, testMappings []mapping) (out []stringPair) {
 						testL := len(rIn)
@@ -386,7 +386,7 @@ var testCasesDoubleEdge = []testCase{
 				orig, replace := e1.orig[0], e1.replace[0]
 				edges := []edge{e1, e2}
 				pairs := buildEdgeTestString(
-					edges, []mapping{getMapping(0), getMapping(m.mask)}, // quote
+					edges, []mapping{getMapping(encoder.EncodeZero), getMapping(m.mask)}, // quote
 					[][]rune{printables, fullwidthPrintables, encodables, encoded, greek}, // fill
 					func(rIn, rOut []rune, quoteOut []bool, testMappings []mapping) (out []stringPair) {
 						testL := len(rIn)
diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go
index 9bcfc90ee3387..0484426c2ff89 100644
--- a/vfs/vfscache/cache.go
+++ b/vfs/vfscache/cache.go
@@ -212,7 +212,7 @@ func (c *Cache) createItemDir(name string) (string, error) {
 
 // getBackend gets a backend for a cache root dir
 func getBackend(ctx context.Context, parentPath string, name string, relativeDirPath string) (fs.Fs, error) {
-	path := fmt.Sprintf("%s/%s/%s", parentPath, name, relativeDirPath)
+	path := fmt.Sprintf(":local,encoding='%v':%s/%s/%s", encoder.OS, parentPath, name, relativeDirPath)
 	return fscache.Get(ctx, path)
 }
 


Test patch:
diff --git a/lib/encoder/encoder_test.go b/lib/encoder/encoder_test.go
index 61f4b73efd652..b3633982418a4 100644
--- a/lib/encoder/encoder_test.go
+++ b/lib/encoder/encoder_test.go
@@ -22,7 +22,7 @@ func TestEncodeString(t *testing.T) {
 		mask MultiEncoder
 		want string
 	}{
-		{0, "None"},
+		{EncodeRaw, "Raw"},
 		{EncodeZero, "None"},
 		{EncodeDoubleQuote, "DoubleQuote"},
 		{EncodeDot, "Dot"},
@@ -44,7 +44,7 @@ func TestEncodeSet(t *testing.T) {
 		wantErr bool
 	}{
 		{"", 0, true},
-		{"None", 0, false},
+		{"Raw", EncodeRaw, false},
 		{"None", EncodeZero, false},
 		{"DoubleQuote", EncodeDoubleQuote, false},
 		{"Dot", EncodeDot, false},
@@ -178,7 +178,7 @@ func TestEncodeInvalidUnicode(t *testing.T) {
 func TestEncodeDot(t *testing.T) {
 	for i, tc := range []testCase{
 		{
-			mask: 0,
+			mask: EncodeZero,
 			in:   ".",
 			out:  ".",
 		}, {
@@ -186,7 +186,7 @@ func TestEncodeDot(t *testing.T) {
 			in:   ".",
 			out:  "．",
 		}, {
-			mask: 0,
+			mask: EncodeZero,
 			in:   "..",
 			out:  "..",
 		}, {
@@ -224,7 +224,7 @@ func TestDecodeHalf(t *testing.T) {
 			in:   "‛",
 			out:  "‛",
 		}, {
-			mask: 0,
+			mask: EncodeZero,
 			in:   "‛‛",
 			out:  "‛",
 		}, {