Repository: rclone/rclone
Pull request #4901 (2021-01-04 16:12:51)

Add SCSU as tables
#### What is the purpose of this change?

Instead of only adding SCSU to #4866, add it as an existing table.

Allow direct SCSU and add a reasonable table as well.

Add byte interfaces that doesn't base64 encode the URL as well with `EncodeBytes` and `DecodeBytes`.

Fuzz tested and decode tests added.

#### Was the change discussed in an issue or in the forum before?

Only in issue #4866

File used for generating the table: [sample.txt.gz](https://github.com/rclone/rclone/files/5765636/sample.txt.gz) using `go run gentable.go -index=sample.txt -scsu -all`

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)


ncw: Sorry for the long delay looking at this!

I think the idea is great and we should merge :-)

The tests indicate we probably need to exclude go1.12 from the build though (we've dropped go1.11 support just recently).
klauspost: @ncw No problem. I don't feel like forking the upstream just for this, so removing 1.12 (which AFAICT has know security issues) would probably be fine.
ncw: Do you want to put some build constraints in to exclude go1.12? That would be fine with me.

The mac build failure is a flakey test

The xgo failure is because the go.mod is out of date WRT the code and go is trying to update it on a read only fs I think.


klauspost: @ncw Updated+master merged.
klauspost: The remaining failures appears to be unrelated now.
ncw: I just added a commit to try to fix the build/other_os - I think what is happening is that the go.mod file isn't sorted so go1.13 is sorting it and can't write it back to the read only docker image...
ncw: If that works, I'll rebase the branch and merge :-)
ncw: OK all built!  I'll merge now - thank you @klauspost 

Next crypt integration and a bit of experimentation.

Patch:
diff --git a/go.mod b/go.mod
index 4f3e6cd64..13c94287a 100644
--- a/go.mod
+++ b/go.mod
@@ -21,6 +21,7 @@ require (
 	github.com/calebcase/tmpfile v1.0.2 // indirect
 	github.com/colinmarc/hdfs/v2 v2.2.0
 	github.com/coreos/go-semver v0.3.0
+	github.com/dop251/scsu v0.0.0-20200422003335-8fadfb689669
 	github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible
 	github.com/gabriel-vasile/mimetype v1.1.2
 	github.com/gogo/protobuf v1.3.2 // indirect
diff --git a/go.sum b/go.sum
index d23f53196..135d8930f 100644
--- a/go.sum
+++ b/go.sum
@@ -171,11 +171,14 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dop251/scsu v0.0.0-20200422003335-8fadfb689669 h1:e28M2/odOZjMc1J2ZZwgex6NM9+aqr1nMlTqPLayxbk=
+github.com/dop251/scsu v0.0.0-20200422003335-8fadfb689669/go.mod h1:Gth7Xev0h28tuTayG4HlTZy90IXhiDgV2+MLtJzjpP0=
 github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible h1:DtumzkLk2zZ2SeElEr+VNz+zV7l+BTe509cV4sKPXbM=
 github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible/go.mod h1:lr+LhMM3F6Y3lW1T9j2U5l7QeuWm87N9+PPXo3yH4qY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813 h1:NgO45/5mBLRVfiXerEFzH6ikcZ7DNRPS639xFg3ENzU=
 github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
diff --git a/lib/encoder/filename/decode.go b/lib/encoder/filename/decode.go
index 75f87ccd5..291d34cc7 100644
--- a/lib/encoder/filename/decode.go
+++ b/lib/encoder/filename/decode.go
@@ -22,6 +22,7 @@ var customDecMu sync.Mutex
 
 // Decode an encoded string.
 func Decode(s string) (string, error) {
+	initCoders()
 	if len(s) < 1 {
 		return "", ErrCorrupted
 	}
@@ -31,19 +32,25 @@ func Decode(s string) (string, error) {
 	}
 	table--
 	s = s[1:]
-
 	data := make([]byte, base64.URLEncoding.DecodedLen(len(s)))
 	n, err := base64.URLEncoding.Decode(data, ([]byte)(s))
 	if err != nil || n < 0 {
 		return "", ErrCorrupted
 	}
 	data = data[:n]
+	return DecodeBytes(table, data)
+}
 
+// DecodeBytes will decode raw id and data values.
+func DecodeBytes(table byte, data []byte) (string, error) {
+	initCoders()
 	switch table {
 	case tableUncompressed:
 		return string(data), nil
 	case tableReserved:
 		return "", ErrUnsupported
+	case tableSCSUPlain:
+		return scsuDecode(data)
 	case tableRLE:
 		if len(data) < 2 {
 			return "", ErrCorrupted
@@ -79,6 +86,9 @@ func Decode(s string) (string, error) {
 		if err != nil {
 			return "", ErrCorrupted
 		}
+		if table == tableSCSU {
+			return scsuDecode(name)
+		}
 		return string(name), nil
 	}
 }
diff --git a/lib/encoder/filename/encode.go b/lib/encoder/filename/encode.go
index 555bf6b91..33d029902 100644
--- a/lib/encoder/filename/encode.go
+++ b/lib/encoder/filename/encode.go
@@ -11,21 +11,45 @@ import (
 // Calling Decode with the returned string should always succeed.
 // It is not a requirement that the input string is valid utf-8.
 func Encode(s string) string {
+	table, payload := EncodeBytes(s)
+	return string(encodeURL[table]) + base64.URLEncoding.EncodeToString(payload)
+}
+
+// EncodeBytes will compress the given string and return a table identifier and a payload.
+func EncodeBytes(s string) (table byte, payload []byte) {
 	initCoders()
 	bestSize := len(s)
-	bestTable := tableUncompressed
+	bestTable := byte(tableUncompressed)
 	org := []byte(s)
 	bestOut := []byte(s)
-
 	// Try all tables and choose the best
 	for i, enc := range encTables[:] {
+		org := org
 		if len(org) <= 1 || len(org) > maxLength {
 			// Use the uncompressed
 			break
 		}
+
 		if enc == nil {
 			continue
 		}
+
+		if i == tableSCSU {
+			var err error
+			olen := len(org)
+			org, err = scsuEncodeStrict(s, make([]byte, 0, len(org)))
+			if err != nil || olen <= len(org) {
+				continue
+			}
+			if len(org) < bestSize {
+				// This is already better, store so we can use if the table cannot.
+				bestOut = bestOut[:len(org)]
+				bestTable = tableSCSUPlain
+				bestSize = len(org)
+				copy(bestOut, org)
+			}
+		}
+
 		// Try to encode using table.
 		err := func() error {
 			encTableLocks[i].Lock()
@@ -36,14 +60,14 @@ func Encode(s string) string {
 			}
 			if len(out) < bestSize {
 				bestOut = bestOut[:len(out)]
-				bestTable = i
+				bestTable = byte(i)
 				bestSize = len(out)
 				copy(bestOut, out)
 			}
 			return nil
 		}()
 		// If input is a single byte repeated store as RLE or save uncompressed.
-		if err == huff0.ErrUseRLE {
+		if err == huff0.ErrUseRLE && i != tableSCSU {
 			if len(org) > 2 {
 				// Encode as one byte repeated since it will be smaller than uncompressed.
 				n := binary.PutUvarint(bestOut, uint64(len(org)))
@@ -56,5 +80,5 @@ func Encode(s string) string {
 		}
 	}
 
-	return string(encodeURL[bestTable]) + base64.URLEncoding.EncodeToString(bestOut)
+	return bestTable, bestOut
 }
diff --git a/lib/encoder/filename/fuzz.go b/lib/encoder/filename/fuzz.go
index 73468ee71..5983a407f 100644
--- a/lib/encoder/filename/fuzz.go
+++ b/lib/encoder/filename/fuzz.go
@@ -25,7 +25,9 @@ func Fuzz(data []byte) int {
 		panic(fmt.Sprintf("error decoding %q, input %q: %v", enc, string(data), err))
 	}
 	if !bytes.Equal(data, []byte(decoded)) {
-		panic(fmt.Sprintf("decode mismatch, encoded: %q, org: %q, got: %q", enc, string(data), decoded))
+		table := decodeMap[enc[0]]
+		table--
+		panic(fmt.Sprintf("decode mismatch, encoded: %q, org: %q, got: %q, table %d", enc, string(data), decoded, int(table)))
 	}
 
 	// Everything is good.
diff --git a/lib/encoder/filename/gentable.go b/lib/encoder/filename/gentable.go
index 4acb61707..5475c60ab 100644
--- a/lib/encoder/filename/gentable.go
+++ b/lib/encoder/filename/gentable.go
@@ -3,21 +3,33 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/base64"
+	"flag"
 	"fmt"
+	"io/ioutil"
 	"math"
+	"strings"
+	"unicode/utf8"
 
+	"github.com/dop251/scsu"
 	"github.com/klauspost/compress"
 	"github.com/klauspost/compress/huff0"
 )
 
-// Replace/add histogram data and execute go run gentable.go
+// execute go run gentable.go
+var indexFile = flag.String("index", "", "Index this file for table")
+
+// Allow non-represented characters.
+var addUnused = flag.Bool("all", true, "Make all bytes possible")
+var scsuEncode = flag.Bool("scsu", false, "SCSU encode on each line before table")
 
 func main() {
-	// Allow non-represented characters.
-	const omitUnused = false
+	flag.Parse()
 
 	histogram := [256]uint64{
+		// Replace/add histogram data and execute go run gentable.go
 		// ncw home directory
 		//0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19442, 760, 0, 349, 570, 1520, 199, 76, 685, 654, 0, 40377, 1605, 395132, 935270, 0, 1156377, 887730, 811737, 712241, 693240, 689139, 675964, 656417, 666577, 657413, 532, 24, 0, 145, 0, 3, 946, 44932, 37362, 46126, 36752, 76346, 19338, 47457, 14288, 38163, 4350, 7867, 36541, 65011, 30255, 26792, 22097, 1803, 39191, 61965, 76585, 11887, 12896, 5931, 1935, 1731, 1385, 1279, 9, 1278, 1, 420185, 0, 1146359, 746359, 968896, 868703, 1393640, 745019, 354147, 159462, 483979, 169092, 75937, 385858, 322166, 466635, 571268, 447132, 13792, 446484, 736844, 732675, 170232, 112983, 63184, 142357, 173945, 21521, 250, 0, 250, 4140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 10, 0, 5, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		//Images:
@@ -26,16 +38,40 @@ func main() {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 7, 0, 0, 0, 7, 1, 1, 0, 2, 1, 506, 706, 0, 3903, 3552, 3694, 3338, 3262, 3257, 3222, 3249, 3325, 3261, 5, 0, 0, 1, 0, 0, 0, 48, 31, 61, 53, 46, 17, 17, 34, 32, 9, 22, 17, 31, 27, 19, 52, 5, 46, 84, 38, 14, 5, 19, 2, 2, 0, 8, 0, 8, 0, 180, 0, 5847, 3282, 3729, 3695, 3842, 3356, 316, 139, 487, 117, 95, 476, 289, 428, 609, 467, 5, 446, 592, 955, 130, 112, 57, 390, 168, 14, 0, 2, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 	}
 
-	// Override with equally distributed characters
-	if false {
-		histogram = [256]uint64{}
-		var chars string
-		// base c64
-		chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
-		// hex
-		//chars = "0123456789abcdef"
-		for _, v := range []byte(chars) {
-			histogram[v] = 1
+	if *indexFile != "" {
+		for i := range histogram[:] {
+			histogram[i] = 0
+		}
+		b, err := ioutil.ReadFile(*indexFile)
+		if err != nil {
+			panic(err)
+		}
+		if *scsuEncode {
+			br := bufio.NewReader(bytes.NewBuffer(b))
+			var encoded []byte
+			for {
+				line, err := br.ReadString('\n')
+				if err != nil {
+					break
+				}
+				line = strings.TrimSpace(line)
+				if len(line) < 3 || !utf8.ValidString(line) {
+					continue
+				}
+				e, err := scsu.Encode(line, nil)
+				if err != nil {
+					panic(err)
+				}
+				if len(e) >= len([]byte(line)) {
+					continue
+				}
+				encoded = append(encoded, e...)
+			}
+			fmt.Println("scsu", len(b), "->", len(encoded), "(excluding bigger)")
+			b = encoded
+		}
+		for _, v := range b {
+			histogram[v]++
 		}
 	}
 
@@ -49,7 +85,7 @@ func main() {
 	const scale = 100 << 10
 	var tmp []byte
 	for i, v := range histogram[:] {
-		if v == 0 && omitUnused {
+		if v == 0 && !*addUnused {
 			continue
 		}
 		nf := float64(v) / float64(total) * scale
diff --git a/lib/encoder/filename/init.go b/lib/encoder/filename/init.go
index 1a7ae41ac..0cbc53c3c 100644
--- a/lib/encoder/filename/init.go
+++ b/lib/encoder/filename/init.go
@@ -26,9 +26,12 @@ var (
 
 const (
 	tableUncompressed = 0
-	tableRLE          = 61
-	tableCustom       = 62
-	tableReserved     = 63
+
+	tableSCSU      = 59
+	tableSCSUPlain = 60
+	tableRLE       = 61
+	tableCustom    = 62
+	tableReserved  = 63
 )
 
 // predefined tables as base64 URL encoded string.
@@ -45,8 +48,16 @@ var tablesData = [64]string{
 	4: "ExDoSTD___-tfXfhJ0hKSkryTxU=",
 	// Base64
 	5: "JRDIcQf_______8PgIiIiIgINkggARHlkQwSSCCBxHFYINHdfXI=",
+	// Hex plus a bit...
+	6: "E5CxwAHm9sYcAlmWZVvMHA4Y5jw=",
+	// Hex, upper case letters.
+	7: "FICxgAMMAGC3YwMthe3DWM_wDAAQ",
 
 	// Special tables:
+	// SCSU and a fairly generic table:
+	tableSCSU: "UxAgZmEB-RYPU8hrnAk6uMgpTNQMB5MGRBx0D3T0JjyUyY-yOi5CoGgktbAktSh7d36HtPTFu7SXJ7FYw_AYmA74ZH2vWgc8O6Z5jLnWnsFqU_4B",
+	// SCSU with no table...
+	tableSCSUPlain: "",
 	// Compressed data has its own table.
 	tableCustom: "",
 	// Reserved for extension.
diff --git a/lib/encoder/filename/scsu.go b/lib/encoder/filename/scsu.go
new file mode 100644
index 000000000..10aaa2b13
--- /dev/null
+++ b/lib/encoder/filename/scsu.go
@@ -0,0 +1,16 @@
+package filename
+
+import "errors"
+
+const scsuNotEnabled = "scsu encoding not enabled in this build due to old go version"
+
+// Functions wrap scsu package, since it doesn't build on old Go versions.
+// Remove once v1.13 is minimum supported version.
+
+var scsuDecode = func(b []byte) (string, error) {
+	return "", errors.New(scsuNotEnabled)
+}
+
+var scsuEncodeStrict = func(src string, dst []byte) ([]byte, error) {
+	return nil, errors.New(scsuNotEnabled)
+}
diff --git a/lib/encoder/filename/scsu_go113.go b/lib/encoder/filename/scsu_go113.go
new file mode 100644
index 000000000..62ee6c75c
--- /dev/null
+++ b/lib/encoder/filename/scsu_go113.go
@@ -0,0 +1,10 @@
+// +build go1.13
+
+package filename
+
+import "github.com/dop251/scsu"
+
+func init() {
+	scsuDecode = scsu.Decode
+	scsuEncodeStrict = scsu.EncodeStrict
+}


Test patch:
diff --git a/lib/encoder/filename/decode_test.go b/lib/encoder/filename/decode_test.go
new file mode 100644
index 000000000..9fff86c5f
--- /dev/null
+++ b/lib/encoder/filename/decode_test.go
@@ -0,0 +1,152 @@
+package filename
+
+import (
+	"runtime"
+	"testing"
+)
+
+func TestDecode(t *testing.T) {
+	tests := []struct {
+		name    string
+		encoded string
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "uncompressed",
+			// tableUncompressed
+			encoded: "AYS5i",
+			want:    "a.b",
+		},
+		{
+			name: "uncompressed-long",
+			// tableUncompressed
+			encoded: "AQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZQnpHUVl4cUhCQTZsalRzaXI4MGdVTTVZ",
+			want:    "BzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5YBzGQYxqHBA6ljTsir80gUM5Y",
+		},
+		{
+			name: "plain-1",
+			// Table 2
+			encoded: "BzGQYxqHBA6ljTsir80gUM5Y=",
+			want:    "-Duplican99E8ZI4___9_",
+		},
+		{
+			name: "hex-1",
+			// Table 4
+			encoded: "D_--tHZROQpqqJ9PafqNa6STF",
+			want:    "13646871dfabbs43323564654bbefff",
+		},
+		{
+			name: "hex-2",
+			// Table 6
+			encoded: "GhIEAIOBQMFQeWm4SClVpXVldCXFZLj4uOgoJHChQ4KBiXQ==",
+			want:    "5368616e6e6f6e206c696d69743a203534353833206279746573-+._=!()",
+		},
+		{
+			name: "hex-3",
+			// Table 7
+			encoded: "HohwXBXoJcVFSHgpdVQlxHXIuVgpNCR06Eg5aBg==",
+			want:    "7461626C6520312073697A653A203335206572723A203C6E696C3E",
+		},
+		{
+			name: "base64-1",
+			// Table 5
+			encoded: "FMpABB9Ef0KP8OrVxjnE3LzUePuLZi8pPg7eW8bgyW2d3Ucckf4rlE0mkAvlILVpOmF3L-rFbmNrpUO2HQFlF4SCMPVPeCEX6LeOg5JVpUVCXV1WSazD9vSpr",
+			want:    "UxAYiB0FNTTkXRw9P8hwq-WmN7tYwbe-sFw8C3snDRG1d-yjrdOUVZQyLdtkJ8tuvhBSnuBiLjVieCAroWEZDIO4Hb_rKgdzPjMqFE7inwHJ2isF==",
+		},
+		{
+			name: "custom-1",
+			// Table 62, custom
+			encoded: "-BeADJCoG_________________xc=",
+			want:    "Uaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		},
+		{
+			name: "custom-2",
+			// Table 62, custom
+			encoded: "-BPABDWUppYyllDKW0sYYSymljJQx",
+			want:    "12312132123121321321321321312312312313132132131231213213213213123121321321321",
+		},
+		{
+			name: "rle-1",
+			// tableRLE
+			encoded: "9a2E=",
+			want:    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		},
+		{
+			name: "regular-1",
+			// Table 1
+			encoded: "BeSSrnzj0j3OXyR9K81M=",
+			want:    "regular-filename.txt",
+		},
+		{
+			name: "regular-3",
+			// Table 2
+			encoded: "COyCCD-42d9s=",
+			want:    "00012312.JPG",
+		},
+		{
+			name: "regular-4",
+			// Table 3
+			encoded: "DmqiJmrhNSDOJTCKTyCQ=",
+			want:    ". . . .txta123123123",
+		},
+		{
+			name: "unicode-1",
+			// tableSCSUPlain
+			encoded: "8D5V3MESVd-WEF7WuqaOvpKUWtYGEyw5UDQ==",
+			want:    "長い長いＵＮＩＣＯＤＥファイル名",
+		},
+		{
+			name: "unicode-2",
+			// tableSCSUPlain
+			encoded: "8GyHV1N7u2OEg4ufQ3eHQ3Ngg6N3X0CDg4-HX0NXU2tg=",
+			want:    "ვეპხის ტყაოსანი შოთა რუსთაველი",
+		},
+		{
+			name: "unicode-3",
+			// tableSCSU
+			encoded: "7LpehMXOrWe7mcT_lpf2MN1Nmgu55jpXHLavZcXJb2UTJ-UmGU15iznkD",
+			want:    "Sønderjysk: Æ ka æe glass uhen at det go mæ naue.,",
+		},
+		{
+			name: "unicode-4",
+			// tableSCSU
+			encoded: "7TCSRm0liJDR0ulpBq4Lla_XB2mWdLFMEs8wEQKHAGa8FRr333ntJ6Ww6_f__N5VKeYM=",
+			want:    "Hello------world     時危兵甲滿天涯，載道流離起怨咨.bin",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := Decode(tt.encoded)
+			if (err != nil) != tt.wantErr {
+				if err != nil && err.Error() == scsuNotEnabled && runtime.Version() < "go1.13" {
+					t.Skip(err.Error())
+					return
+				}
+				if tt.encoded == "" && tt.want != "" {
+					proposed := Encode(tt.want)
+					table := decodeMap[proposed[0]] - 1
+					t.Errorf("No encoded value, try '%s', table is %d", proposed, table)
+					return
+				}
+				t.Errorf("Decode() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if err == nil {
+				proposed := Encode(tt.want)
+				table := decodeMap[proposed[0]] - 1
+				if len(proposed) > len(tt.encoded) {
+					t.Errorf("Got longer encoded value than reference. Likely compression regression. Got %s, table %d", proposed, table)
+				}
+				if len(proposed) > len(tt.encoded) {
+					t.Logf("Got better encoded value, improved length %d, was %d", len(proposed), len(tt.encoded))
+				}
+			}
+
+			if got != tt.want {
+				t.Errorf("Decode() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}