Repository: rclone/rclone
Pull request #5345 (2021-05-21 09:29:46)

Warning about `too many open files`
Fix https://github.com/rclone/rclone/issues/5310

<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?

<!--
Describe the changes here
-->

#### Was the change discussed in an issue or in the forum before?

<!--
Link issues and relevant forum posts here.
-->

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [ ] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go
index 4a996806e..c007d0b33 100755
--- a/backend/dropbox/dropbox.go
+++ b/backend/dropbox/dropbox.go
@@ -138,7 +138,7 @@ func getOauthConfig(m configmap.Mapper) *oauth2.Config {
 
 // Register with Fs
 func init() {
-	DbHashType = hash.RegisterHash("DropboxHash", 64, dbhash.New)
+	DbHashType = hash.RegisterHash("dropbox", "DropboxHash", 64, dbhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "dropbox",
 		Description: "Dropbox",
diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go
index 0398dea32..8a437cf37 100644
--- a/backend/mailru/mailru.go
+++ b/backend/mailru/mailru.go
@@ -80,7 +80,7 @@ var oauthConfig = &oauth2.Config{
 
 // Register with Fs
 func init() {
-	MrHashType = hash.RegisterHash("MailruHash", 40, mrhash.New)
+	MrHashType = hash.RegisterHash("mailru", "MailruHash", 40, mrhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "mailru",
 		Description: "Mail.ru Cloud",
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go
index 16c1225fe..611960a54 100755
--- a/backend/onedrive/onedrive.go
+++ b/backend/onedrive/onedrive.go
@@ -93,7 +93,7 @@ var (
 
 // Register with Fs
 func init() {
-	QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New)
+	QuickXorHashType = hash.RegisterHash("quickxor", "QuickXorHash", 40, quickxorhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "onedrive",
 		Description: "Microsoft OneDrive",
diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go
index ce397611c..a69f8633b 100644
--- a/cmd/hashsum/hashsum.go
+++ b/cmd/hashsum/hashsum.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/cmd"
@@ -69,23 +70,17 @@ hashed locally enabling any hash for any remote.
 Run without a hash to see the list of all supported hashes, e.g.
 
     $ rclone hashsum
-    Supported hashes are:
-      * MD5
-      * SHA-1
-      * DropboxHash
-      * QuickXorHash
-
+` + hashListHelp("    ") + `
 Then
 
     $ rclone hashsum MD5 remote:path
+
+Note that hash names are case insensitive.
 `,
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 2, command, args)
 		if len(args) == 0 {
-			fmt.Printf("Supported hashes are:\n")
-			for _, ht := range hash.Supported().Array() {
-				fmt.Printf("  * %v\n", ht)
-			}
+			fmt.Print(hashListHelp(""))
 			return nil
 		} else if len(args) == 1 {
 			return errors.New("need hash type and remote")
@@ -93,6 +88,7 @@ Then
 		var ht hash.Type
 		err := ht.Set(args[0])
 		if err != nil {
+			fmt.Println(hashListHelp(""))
 			return err
 		}
 		fsrc := cmd.NewFsSrc(args[1:])
@@ -111,3 +107,14 @@ Then
 		return nil
 	},
 }
+
+func hashListHelp(indent string) string {
+	var help strings.Builder
+	help.WriteString(indent)
+	help.WriteString("Supported hashes are:\n")
+	for _, ht := range hash.Supported().Array() {
+		help.WriteString(indent)
+		fmt.Fprintf(&help, "  * %v\n", ht.String())
+	}
+	return help.String()
+}
diff --git a/docs/content/commands/rclone_hashsum.md b/docs/content/commands/rclone_hashsum.md
index 2f150c69a..45cddc2e4 100644
--- a/docs/content/commands/rclone_hashsum.md
+++ b/docs/content/commands/rclone_hashsum.md
@@ -25,15 +25,20 @@ Run without a hash to see the list of all supported hashes, e.g.
 
     $ rclone hashsum
     Supported hashes are:
-      * MD5
-      * SHA-1
-      * DropboxHash
-      * QuickXorHash
+      * md5
+      * sha1
+      * whirlpool
+      * crc32
+      * dropbox
+      * mailru
+      * quickxor
 
 Then
 
     $ rclone hashsum MD5 remote:path
 
+Note that hash names are case insensitive.
+
 
 ```
 rclone hashsum <hash> remote:path [flags]
diff --git a/docs/content/flags.md b/docs/content/flags.md
old mode 100755
new mode 100644
diff --git a/docs/content/tardigrade.md b/docs/content/tardigrade.md
index ca448b0d2..15d7fb174 100644
--- a/docs/content/tardigrade.md
+++ b/docs/content/tardigrade.md
@@ -310,3 +310,8 @@ remote.
 See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
 See [rclone about](https://rclone.org/commands/rclone_about/)
 
+### Known issues
+
+If you get errors like `too many open files` this usually happens when the default `ulimit` for system max open files is exceeded. Native Storj protocol opens a large number of TCP connections (each of which is counted as an open file). For a single upload stream you can expect 110 TCP connections to be opened. For a single download stream you can expect 35. This batch of connections will be opened for every 64 MiB segment and you should also expect TCP connections to be reused. If you do many transfers you eventually open a connection to most storage nodes (thousands of nodes).
+
+To fix these, please raise your system limits. You can do this issuing a `ulimit -n 65536` just before you run rclone, inside a bash script or inside the `$USER/.bashrc` file. If you need to change limits system-wide, this usually takes place at `/etc/sysctl.conf` and/or `/etc/security/limits.conf` on Linux, but please refer to your operating system manual.
diff --git a/fs/hash/hash.go b/fs/hash/hash.go
index 8b49c2476..e0d791d95 100644
--- a/fs/hash/hash.go
+++ b/fs/hash/hash.go
@@ -20,25 +20,36 @@ type Type int
 type hashDefinition struct {
 	width    int
 	name     string
+	alias    string
 	newFunc  func() hash.Hash
 	hashType Type
 }
 
-var hashes []*hashDefinition
-var highestType Type = 1
+var (
+	type2hash  = map[Type]*hashDefinition{}
+	name2hash  = map[string]*hashDefinition{}
+	alias2hash = map[string]*hashDefinition{}
+	supported  = []Type{}
+)
 
 // RegisterHash adds a new Hash to the list and returns it Type
-func RegisterHash(name string, width int, newFunc func() hash.Hash) Type {
+func RegisterHash(name, alias string, width int, newFunc func() hash.Hash) Type {
+	hashType := Type(1 << len(supported))
+	supported = append(supported, hashType)
+
 	definition := &hashDefinition{
 		name:     name,
+		alias:    alias,
 		width:    width,
 		newFunc:  newFunc,
-		hashType: highestType,
+		hashType: hashType,
 	}
-	hashes = append(hashes, definition)
-	highestType = highestType << 1
 
-	return definition.hashType
+	type2hash[hashType] = definition
+	name2hash[name] = definition
+	alias2hash[alias] = definition
+
+	return hashType
 }
 
 // ErrUnsupported should be returned by filesystem,
@@ -63,31 +74,23 @@ var (
 )
 
 func init() {
-	MD5 = RegisterHash("MD5", 32, md5.New)
-	SHA1 = RegisterHash("SHA-1", 40, sha1.New)
-	Whirlpool = RegisterHash("Whirlpool", 128, whirlpool.New)
-	CRC32 = RegisterHash("CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() })
+	MD5 = RegisterHash("md5", "MD5", 32, md5.New)
+	SHA1 = RegisterHash("sha1", "SHA-1", 40, sha1.New)
+	Whirlpool = RegisterHash("whirlpool", "Whirlpool", 128, whirlpool.New)
+	CRC32 = RegisterHash("crc32", "CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() })
 }
 
 // Supported returns a set of all the supported hashes by
 // HashStream and MultiHasher.
 func Supported() Set {
-	var types []Type
-	for _, v := range hashes {
-		types = append(types, v.hashType)
-	}
-
-	return NewHashSet(types...)
+	return NewHashSet(supported...)
 }
 
 // Width returns the width in characters for any HashType
 func Width(hashType Type) int {
-	for _, v := range hashes {
-		if v.hashType == hashType {
-			return v.width
-		}
+	if hash := type2hash[hashType]; hash != nil {
+		return hash.width
 	}
-
 	return 0
 }
 
@@ -118,32 +121,29 @@ func StreamTypes(r io.Reader, set Set) (map[Type]string, error) {
 // The function will panic if the hash type is unknown.
 func (h Type) String() string {
 	if h == None {
-		return "None"
+		return "none"
 	}
-
-	for _, v := range hashes {
-		if v.hashType == h {
-			return v.name
-		}
+	if hash := type2hash[h]; hash != nil {
+		return hash.name
 	}
-
-	err := fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h))
-	panic(err)
+	panic(fmt.Sprintf("internal error: unknown hash type: 0x%x", int(h)))
 }
 
-// Set a Type from a flag
+// Set a Type from a flag.
+// Both name and alias are accepted.
 func (h *Type) Set(s string) error {
-	if s == "None" {
+	if s == "none" || s == "None" {
 		*h = None
+		return nil
 	}
-
-	for _, v := range hashes {
-		if v.name == s {
-			*h = v.hashType
-			return nil
-		}
+	if hash := name2hash[strings.ToLower(s)]; hash != nil {
+		*h = hash.hashType
+		return nil
+	}
+	if hash := alias2hash[s]; hash != nil {
+		*h = hash.hashType
+		return nil
 	}
-
 	return errors.Errorf("Unknown hash type %q", s)
 }
 
@@ -159,23 +159,14 @@ func fromTypes(set Set) (map[Type]hash.Hash, error) {
 	if !set.SubsetOf(Supported()) {
 		return nil, errors.Errorf("requested set %08x contains unknown hash types", int(set))
 	}
-	var hashers = make(map[Type]hash.Hash)
-
-	types := set.Array()
-	for _, t := range types {
-		for _, v := range hashes {
-			if t != v.hashType {
-				continue
-			}
-
-			hashers[t] = v.newFunc()
-			break
-		}
+	hashers := map[Type]hash.Hash{}
 
-		if hashers[t] == nil {
-			err := fmt.Sprintf("internal error: Unsupported hash type %v", t)
-			panic(err)
+	for _, t := range set.Array() {
+		hash := type2hash[t]
+		if hash == nil {
+			panic(fmt.Sprintf("internal error: Unsupported hash type %v", t))
 		}
+		hashers[t] = hash.newFunc()
 	}
 
 	return hashers, nil


Test patch:
diff --git a/fs/hash/hash_test.go b/fs/hash/hash_test.go
index e7653dbb1..722e5d898 100644
--- a/fs/hash/hash_test.go
+++ b/fs/hash/hash_test.go
@@ -156,16 +156,53 @@ func TestHashStreamTypes(t *testing.T) {
 
 func TestHashSetStringer(t *testing.T) {
 	h := hash.NewHashSet(hash.SHA1, hash.MD5)
-	assert.Equal(t, h.String(), "[MD5, SHA-1]")
+	assert.Equal(t, "[md5, sha1]", h.String())
 	h = hash.NewHashSet(hash.SHA1)
-	assert.Equal(t, h.String(), "[SHA-1]")
+	assert.Equal(t, "[sha1]", h.String())
 	h = hash.NewHashSet()
-	assert.Equal(t, h.String(), "[]")
+	assert.Equal(t, "[]", h.String())
 }
 
 func TestHashStringer(t *testing.T) {
 	h := hash.MD5
-	assert.Equal(t, h.String(), "MD5")
+	assert.Equal(t, "md5", h.String())
+	h = hash.SHA1
+	assert.Equal(t, "sha1", h.String())
 	h = hash.None
-	assert.Equal(t, h.String(), "None")
+	assert.Equal(t, "none", h.String())
+}
+
+func TestHashSetter(t *testing.T) {
+	var ht hash.Type
+
+	assert.NoError(t, ht.Set("none"))
+	assert.Equal(t, hash.None, ht)
+	assert.NoError(t, ht.Set("None"))
+	assert.Equal(t, hash.None, ht)
+
+	assert.NoError(t, ht.Set("md5"))
+	assert.Equal(t, hash.MD5, ht)
+	assert.NoError(t, ht.Set("MD5"))
+	assert.Equal(t, hash.MD5, ht)
+
+	assert.NoError(t, ht.Set("sha1"))
+	assert.Equal(t, hash.SHA1, ht)
+	assert.NoError(t, ht.Set("SHA-1"))
+	assert.Equal(t, hash.SHA1, ht)
+
+	assert.NoError(t, ht.Set("SHA1"))
+	assert.Equal(t, hash.SHA1, ht)
+	assert.NoError(t, ht.Set("Sha1"))
+	assert.Equal(t, hash.SHA1, ht)
+	assert.Error(t, ht.Set("Sha-1"))
+}
+
+func TestHashTypeStability(t *testing.T) {
+	assert.Equal(t, hash.Type(0), hash.None)
+	assert.Equal(t, hash.Type(1), hash.MD5)
+	assert.Equal(t, hash.Type(2), hash.SHA1)
+
+	assert.True(t, hash.Supported().Contains(hash.MD5))
+	assert.True(t, hash.Supported().Contains(hash.SHA1))
+	assert.False(t, hash.Supported().Contains(hash.None))
 }
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 603314b7d..f01b07a3e 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1206,10 +1206,10 @@ func TestListFormat(t *testing.T) {
 			Format: "2006-01-02T15:04:05.000000000Z07:00"},
 		IsDir: false,
 		Hashes: map[string]string{
-			"MD5":          "0cc175b9c0f1b6a831c399e269772661",
-			"SHA-1":        "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8",
-			"DropboxHash":  "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8",
-			"QuickXorHash": "6100000000000000000000000100000000000000"},
+			"md5":      "0cc175b9c0f1b6a831c399e269772661",
+			"sha1":     "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8",
+			"dropbox":  "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8",
+			"quickxor": "6100000000000000000000000100000000000000"},
 		ID:     "fileID",
 		OrigID: "fileOrigID",
 	}
diff --git a/fs/options_test.go b/fs/options_test.go
index d02531078..52fc42ddb 100644
--- a/fs/options_test.go
+++ b/fs/options_test.go
@@ -115,7 +115,7 @@ func TestHTTPOption(t *testing.T) {
 func TestHashesOption(t *testing.T) {
 	opt := &HashesOption{hash.Set(hash.MD5 | hash.SHA1)}
 	var _ OpenOption = opt // check interface
-	assert.Equal(t, `HashesOption([MD5, SHA-1])`, opt.String())
+	assert.Equal(t, `HashesOption([md5, sha1])`, opt.String())
 	key, value := opt.Header()
 	assert.Equal(t, "", key)
 	assert.Equal(t, "", value)