Repository: rclone/rclone
Pull request #7134 (2023-07-12 19:34:26)

s3: Add Leviia S3 Object Storage as provider
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.

You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->

#### What is the purpose of this change?

<!--
Add Leviia, French S3 Object storage, as provider.
-->

#### Was the change discussed in an issue or in the forum before?

<!--
[Link issues and relevant forum posts here.](https://forum.rclone.org/t/add-leviia-as-provider-for-s3-object-storage/39863)
-->

#### Checklist

- [x] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [x] I have added tests for all changes in this PR if appropriate.
- [x] I have added documentation for the changes if appropriate.
- [x] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [x] I'm done, this Pull Request is ready for review :-)



Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b855794bf..2ac54c20e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,12 +27,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.18', 'go1.19']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.19', 'go1.20']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.21.0-rc.3'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -43,14 +43,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.21.0-rc.3'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-11
-            go: '1.20'
+            go: '1.21.0-rc.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -59,14 +59,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.20'
+            go: '1.21.0-rc.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '1.20'
+            go: '1.21.0-rc.3'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -76,20 +76,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.21.0-rc.3'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.18
+          - job_name: go1.19
             os: ubuntu-latest
-            go: '1.18'
+            go: '1.19'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.19
+          - job_name: go1.20
             os: ubuntu-latest
-            go: '1.19'
+            go: '1.20'
             quicktest: true
             racequicktest: true
 
@@ -130,6 +130,11 @@ jobs:
       - name: Install Libraries on macOS
         shell: bash
         run: |
+          # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
+          # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
+          unset HOMEBREW_NO_INSTALL_FROM_API
+          brew untap --force homebrew/core
+          brew untap --force homebrew/cask
           brew update
           brew install --cask macfuse
         if: matrix.os == 'macos-11'
@@ -239,7 +244,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21.0-rc.3'
           check-latest: true
 
       - name: Install govulncheck
@@ -264,7 +269,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21.0-rc.3'
 
       - name: Go module cache
         uses: actions/cache@v3
diff --git a/README.md b/README.md
index 221041854..77cea8918 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
   * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
+  * Leviia [:page_facing_up:](https://rclone.org/s3/#leviia)
   * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 149938b6e..2491efebc 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -1,5 +1,5 @@
-//go:build !plan9 && !solaris && !js && go1.18
-// +build !plan9,!solaris,!js,go1.18
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
diff --git a/backend/azureblob/azureblob_unsupported.go b/backend/azureblob/azureblob_unsupported.go
index 08d5c4063..369d6e367 100644
--- a/backend/azureblob/azureblob_unsupported.go
+++ b/backend/azureblob/azureblob_unsupported.go
@@ -1,7 +1,7 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || solaris || js || !go1.18
-// +build plan9 solaris js !go1.18
+//go:build plan9 || solaris || js
+// +build plan9 solaris js
 
 package azureblob
diff --git a/backend/box/box.go b/backend/box/box.go
index 57e8d4f23..e932b0762 100644
--- a/backend/box/box.go
+++ b/backend/box/box.go
@@ -77,7 +77,7 @@ var (
 )
 
 type boxCustomClaims struct {
-	jwt.RegisteredClaims
+	jwt.StandardClaims
 	BoxSubType string `json:"box_sub_type,omitempty"`
 }
 
@@ -208,12 +208,14 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
 	}
 
 	claims = &boxCustomClaims{
-		RegisteredClaims: jwt.RegisteredClaims{
-			ID:        val,
+		//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
+		//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
+		StandardClaims: jwt.StandardClaims{
+			Id:        val,
 			Issuer:    boxConfig.BoxAppSettings.ClientID,
 			Subject:   boxConfig.EnterpriseID,
-			Audience:  jwt.ClaimStrings{tokenURL},
-			ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
+			Audience:  tokenURL,
+			ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
 		},
 		BoxSubType: boxSubType,
 	}
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index d51702279..6c0cf435c 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -66,7 +66,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, China Mobile, Cloudflare, GCS, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, China Mobile, Cloudflare, GCS, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Leviia, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
@@ -127,6 +127,9 @@ func init() {
 			}, {
 				Value: "LyveCloud",
 				Help:  "Seagate Lyve Cloud",
+			}, {
+				Value: "Leviia",
+				Help:  "Leviia Object Storage",
 			}, {
 				Value: "Liara",
 				Help:  "Liara Object Storage",
@@ -840,6 +843,15 @@ func init() {
 				Value: "s3.sa-east-1.petabox.io",
 				Help:  "South America (São Paulo)",
 			}},
+		}, {
+			// Leviia endpoints: https://www.leviia.com/object-storage/
+			Name:     "endpoint",
+			Help:     "Endpoint for Leviia Object Storage API.",
+			Provider: "Leviia",
+			Examples: []fs.OptionExample{{
+				Value: "s3.leviia.com",
+				Help:  "The default endpoint\nLeviia",
+			}},
 		}, {
 			// Liara endpoints: https://liara.ir/landing/object-storage
 			Name:     "endpoint",
@@ -1689,7 +1701,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS,Petabox",
+			Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS,Petabox",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -3026,6 +3038,8 @@ func setQuirks(opt *Options) {
 		useMultipartEtag = false // untested
 	case "Wasabi":
 		// No quirks
+	case "Leviia":
+		// No quirks
 	case "Qiniu":
 		useMultipartEtag = false
 		urlEncodeListings = false
diff --git a/docs/content/authors.md b/docs/content/authors.md
index 2e9e2257e..1f468acaa 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -744,3 +744,4 @@ put them back in again.` >}}
   * BakaWang <wa11579@hotmail.com>
   * Mahad <56235065+Mahad-lab@users.noreply.github.com>
   * Vladislav Vorobev <x.miere@gmail.com>
+  * darix <darix@users.noreply.github.com>
diff --git a/docs/content/s3.md b/docs/content/s3.md
index e379f7f34..9d7d0522c 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -22,6 +22,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
 {{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}}
 {{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}}
+ {{< provider name="Leviia Object Storage" home="https://www.leviia.com/object-storage/" config="/s3/#leviia" >}}
 {{< provider name="Liara Object Storage" home="https://liara.ir/landing/object-storage" config="/s3/#liara-cloud" >}}
 {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
 {{< provider name="Petabox" home="https://petabox.io/" config="/s3/#petabox" >}}
@@ -5158,7 +5159,129 @@ e) Edit this remote
 d) Delete this remote
 y/e/d> y
 ```
+### Leviia Cloud Object Storage {#leviia}
 
+[Leviia Object Storage](https://www.leviia.com/object-storage/), backup and secure your data in a 100% French cloud, independent of GAFAM..
+
+To configure access to Leviia, follow the steps below:
+
+1. Run `rclone config` and select `n` for a new remote.
+
+```
+rclone config
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+```
+
+2. Give the name of the configuration. For example, name it 'leviia'.
+
+```
+name> leviia
+```
+
+3. Select `s3` storage.
+
+```
+Choose a number from below, or type in your own value
+ 1 / 1Fichier
+   \ (fichier)
+ 2 / Akamai NetStorage
+   \ (netstorage)
+ 3 / Alias for an existing remote
+   \ (alias)
+ 4 / Amazon Drive
+   \ (amazon cloud drive)
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi
+   \ (s3)
+[snip]
+Storage> s3
+```
+
+4. Select `Leviia` provider.
+```
+Choose a number from below, or type in your own value
+1 / Amazon Web Services (AWS) S3
+   \ "AWS"
+[snip]
+15 / Leviia Object Storage
+   \ (Leviia)
+[snip]
+provider> Leviia
+```
+
+5. Enter your SecretId and SecretKey of Leviia.
+
+```
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Enter a boolean value (true or false). Press Enter for the default ("false").
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+access_key_id> ZnIx.xxxxxxxxxxxxxxx
+AWS Secret Access Key (password)
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+secret_access_key> xxxxxxxxxxx
+```
+
+6. Select endpoint for Leviia.
+
+```
+   / The default endpoint
+ 1 | Leviia.
+   \ (s3.leviia.com)
+[snip]
+endpoint> 1
+
+7. Choose acl.
+
+```
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+ 2 | The AllUsers group gets READ access.
+   \ (public-read)
+[snip]
+acl> 1
+Edit advanced config? (y/n)
+y) Yes
+n) No (default)
+y/n> n
+Remote config
+--------------------
+[leviia]
+- type: s3
+- provider: Leviia
+- access_key_id: ZnIx.xxxxxxx
+- secret_access_key: xxxxxxxx
+- endpoint: s3.leviia.com
+- acl: private
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+leviia                s3
+```
 ### Liara {#liara-cloud}
 
 Here is an example of making a [Liara Object Storage](https://liara.ir/landing/object-storage)
diff --git a/fs/rc/internal.go b/fs/rc/internal.go
index 4dafbb506..e2f439a29 100644
--- a/fs/rc/internal.go
+++ b/fs/rc/internal.go
@@ -389,10 +389,7 @@ func rcSetSoftMemoryLimit(ctx context.Context, in Params) (out Params, err error
 	if err != nil {
 		return nil, err
 	}
-	oldMemLimit, err := debug.SetMemoryLimit(memLimit)
-	if err != nil {
-		return nil, err
-	}
+	oldMemLimit := debug.SetMemoryLimit(memLimit)
 	out = Params{
 		"existing-mem-limit": oldMemLimit,
 	}
diff --git a/fs/versioncheck.go b/fs/versioncheck.go
index 7a58a4ac5..afa274c92 100644
--- a/fs/versioncheck.go
+++ b/fs/versioncheck.go
@@ -1,8 +1,8 @@
-//go:build !go1.18
-// +build !go1.18
+//go:build !go1.19
+// +build !go1.19
 
 package fs
 
-// Upgrade to Go version 1.18 to compile rclone - latest stable go
+// Upgrade to Go version 1.19 to compile rclone - latest stable go
 // compiler recommended.
-func init() { Go_version_1_18_required_for_compilation() }
+func init() { Go_version_1_19_required_for_compilation() }
diff --git a/go.mod b/go.mod
index 8c87c7cc7..99136dc17 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/rclone/rclone
 
-go 1.18
+go 1.19
 
 require (
 	bazil.org/fuse v0.0.0-20221209211307-2abb8038c751
diff --git a/lib/debug/common.go b/lib/debug/common.go
index ab0a28da9..14ea9ddc2 100644
--- a/lib/debug/common.go
+++ b/lib/debug/common.go
@@ -10,3 +10,9 @@ import (
 func SetGCPercent(percent int) int {
 	return debug.SetGCPercent(percent)
 }
+
+// SetMemoryLimit calls the runtime/debug.SetMemoryLimit function to set the
+// soft-memory limit.
+func SetMemoryLimit(limit int64) int64 {
+	return debug.SetMemoryLimit(limit)
+}
diff --git a/lib/debug/go1.19.go b/lib/debug/go1.19.go
deleted file mode 100644
index 1103f2025..000000000
--- a/lib/debug/go1.19.go
+++ /dev/null
@@ -1,14 +0,0 @@
-//go:build go1.19
-// +build go1.19
-
-package debug
-
-import (
-	"runtime/debug"
-)
-
-// SetMemoryLimit calls the runtime/debug.SetMemoryLimit function to set the
-// soft-memory limit.
-func SetMemoryLimit(limit int64) (int64, error) {
-	return debug.SetMemoryLimit(limit), nil
-}
diff --git a/lib/debug/go1.19_compat.go b/lib/debug/go1.19_compat.go
deleted file mode 100644
index e4ccbc695..000000000
--- a/lib/debug/go1.19_compat.go
+++ /dev/null
@@ -1,14 +0,0 @@
-//go:build !go1.19
-// +build !go1.19
-
-package debug
-
-import (
-	"fmt"
-	"runtime"
-)
-
-// SetMemoryLimit is a no-op on Go version < 1.19.
-func SetMemoryLimit(limit int64) (int64, error) {
-	return limit, fmt.Errorf("not implemented on Go version below 1.19: %s", runtime.Version())
-}
diff --git a/lib/rest/rest.go b/lib/rest/rest.go
index f248d441d..c4fb51b6a 100644
--- a/lib/rest/rest.go
+++ b/lib/rest/rest.go
@@ -11,7 +11,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/url"
@@ -163,7 +162,7 @@ const drainLimit = 10 * 1024 * 1024
 // drainAndClose discards up to drainLimit bytes from r and closes
 // it. Any errors from the Read or Close are returned.
 func drainAndClose(r io.ReadCloser) (err error) {
-	_, readErr := io.CopyN(ioutil.Discard, r, drainLimit)
+	_, readErr := io.CopyN(io.Discard, r, drainLimit)
 	if readErr == io.EOF {
 		readErr = nil
 	}


Test patch:
diff --git a/backend/azureblob/azureblob_internal_test.go b/backend/azureblob/azureblob_internal_test.go
index daa3b8114..827653a80 100644
--- a/backend/azureblob/azureblob_internal_test.go
+++ b/backend/azureblob/azureblob_internal_test.go
@@ -1,5 +1,5 @@
-//go:build !plan9 && !solaris && !js && go1.18
-// +build !plan9,!solaris,!js,go1.18
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 
diff --git a/backend/azureblob/azureblob_test.go b/backend/azureblob/azureblob_test.go
index a67b71ad6..f6122b892 100644
--- a/backend/azureblob/azureblob_test.go
+++ b/backend/azureblob/azureblob_test.go
@@ -1,7 +1,7 @@
 // Test AzureBlob filesystem interface
 
-//go:build !plan9 && !solaris && !js && go1.18
-// +build !plan9,!solaris,!js,go1.18
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 
diff --git a/backend/cache/cache_upload_test.go b/backend/cache/cache_upload_test.go
index b6a2c5842..d0246740a 100644
--- a/backend/cache/cache_upload_test.go
+++ b/backend/cache/cache_upload_test.go
@@ -160,11 +160,11 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	minSize := 5242880
 	maxSize := 10485760
 	totalFiles := 10
-	rand.Seed(time.Now().Unix())
+	randInstance := rand.New(rand.NewSource(time.Now().Unix()))
 
 	lastFile := ""
 	for i := 0; i < totalFiles; i++ {
-		size := int64(rand.Intn(maxSize-minSize) + minSize)
+		size := int64(randInstance.Intn(maxSize-minSize) + minSize)
 		testReader := runInstance.randomReader(t, size)
 		remote := "test/" + strconv.Itoa(i) + ".bin"
 		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
diff --git a/fstest/test_all/test_all.go b/fstest/test_all/test_all.go
index 1bf0311e0..d536b83dd 100644
--- a/fstest/test_all/test_all.go
+++ b/fstest/test_all/test_all.go
@@ -73,7 +73,7 @@ func main() {
 	configfile.Install()
 
 	// Seed the random number generator
-	rand.Seed(time.Now().UTC().UnixNano())
+	randInstance := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
 
 	// Filter selection
 	if *testRemotes != "" {
@@ -103,7 +103,7 @@ func main() {
 
 	// Runs we will do for this test in random order
 	runs := conf.MakeRuns()
-	rand.Shuffle(len(runs), runs.Swap)
+	randInstance.Shuffle(len(runs), runs.Swap)
 
 	// Create Report
 	report := NewReport()