Repository: truenas/middleware
Pull request #16479: NAS-133835 / 25.10 / Allow setting Storj endpoint for global and select tiers
Author: login: creatorcary
Labels: name: backported
name: no-time-tracked
name: backport-25.10.1
Created: 2025-05-12T15:37:28Z | Merged: 2025-05-13T15:23:50Z

Existing and future Storj credentials will default to using https://gateway.storjshare.io/.

UI follow-up ticket will be needed.


Diff:
diff --git a/src/middlewared/middlewared/alembic/versions/25.04/2025-05-13_20-47_storj_endpoint.py b/src/middlewared/middlewared/alembic/versions/25.04/2025-05-13_20-47_storj_endpoint.py
new file mode 100644
index 0000000000000..d6d11f102075c
--- /dev/null
+++ b/src/middlewared/middlewared/alembic/versions/25.04/2025-05-13_20-47_storj_endpoint.py
@@ -0,0 +1,30 @@
+"""Add `endpoint` field to system_cloudcredentials.attributes for type STORJ_IX.
+
+Revision ID: 30c9619bf9e7
+Revises: 7c663f5ec8a1
+Create Date: 2025-05-07 20:47:46.751451+00:00
+
+"""
+import json
+
+from alembic import op
+
+from middlewared.plugins.pwenc import encrypt, decrypt
+
+
+# revision identifiers, used by Alembic.
+revision = '30c9619bf9e7'
+down_revision = '7c663f5ec8a1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    conn = op.get_bind()
+    for entry in map(dict, conn.execute("SELECT id, attributes FROM system_cloudcredentials WHERE provider = 'STORJ_IX'").fetchall()):
+        if attributes := decrypt(entry["attributes"]):
+            attributes = json.loads(attributes)
+            attributes.setdefault("endpoint", "https://gateway.storjshare.io/")
+            conn.execute("UPDATE system_cloudcredentials SET attributes = ? WHERE id = ?", (
+                encrypt(json.dumps(attributes)), entry["id"]
+            ))
diff --git a/src/middlewared/middlewared/alembic/versions/25.10/2025-05-13_12-00_merge.py b/src/middlewared/middlewared/alembic/versions/25.10/2025-05-13_12-00_merge.py
index 3a5fc9fad2c18..e0b2a10433c59 100644
--- a/src/middlewared/middlewared/alembic/versions/25.10/2025-05-13_12-00_merge.py
+++ b/src/middlewared/middlewared/alembic/versions/25.10/2025-05-13_12-00_merge.py
@@ -1,7 +1,7 @@
 """Merge
 
 Revision ID: c227e49be4d8
-Revises: aeae837fb2d5, 7c663f5ec8a1
+Revises: aeae837fb2d5, 30c9619bf9e7
 Create Date: 2025-05-13 12:00:55.304980+00:00
 
 """
@@ -11,7 +11,7 @@
 
 # revision identifiers, used by Alembic.
 revision = 'c227e49be4d8'
-down_revision = ('aeae837fb2d5', '7c663f5ec8a1')
+down_revision = ('aeae837fb2d5', '30c9619bf9e7')
 branch_labels = None
 depends_on = None
 
diff --git a/src/middlewared/middlewared/api/v25_04_2/cloud_sync_providers.py b/src/middlewared/middlewared/api/v25_04_2/cloud_sync_providers.py
index 59d6f8f650643..1144b4485226b 100644
--- a/src/middlewared/middlewared/api/v25_04_2/cloud_sync_providers.py
+++ b/src/middlewared/middlewared/api/v25_04_2/cloud_sync_providers.py
@@ -3,7 +3,7 @@
 
 from pydantic import AfterValidator, Discriminator, Field, Secret
 
-from middlewared.api.base import BaseModel, HttpUrl, LongNonEmptyString, match_validator, NonEmptyString
+from middlewared.api.base import BaseModel, HttpUrl, LongNonEmptyString, match_validator, NonEmptyString, HttpsOnlyURL
 
 __all__ = ["CloudCredentialProvider"]
 
@@ -122,6 +122,7 @@ class StorjIxCredentialsModel(BaseModel):
     type: Literal["STORJ_IX"]
     access_key_id: Secret[NonEmptyString]
     secret_access_key: Secret[NonEmptyString]
+    endpoint: HttpsOnlyURL = "https://gateway.storjshare.io/"
 
 
 class SwiftCredentialsModel(BaseModel):
diff --git a/src/middlewared/middlewared/api/v25_10_0/cloud_sync_providers.py b/src/middlewared/middlewared/api/v25_10_0/cloud_sync_providers.py
index 73e2d183bf7c0..ab64eb508d906 100644
--- a/src/middlewared/middlewared/api/v25_10_0/cloud_sync_providers.py
+++ b/src/middlewared/middlewared/api/v25_10_0/cloud_sync_providers.py
@@ -3,7 +3,7 @@
 
 from pydantic import AfterValidator, Discriminator, Field, Secret
 
-from middlewared.api.base import BaseModel, HttpUrl, LongNonEmptyString, match_validator, NonEmptyString
+from middlewared.api.base import BaseModel, HttpUrl, LongNonEmptyString, match_validator, NonEmptyString, HttpsOnlyURL
 
 __all__ = ["CloudCredentialProvider"]
 
@@ -128,6 +128,7 @@ class StorjIxCredentialsModel(BaseModel):
     type: Literal["STORJ_IX"]
     access_key_id: Secret[NonEmptyString]
     secret_access_key: Secret[NonEmptyString]
+    endpoint: HttpsOnlyURL = "https://gateway.storjshare.io/"
 
 
 class SwiftCredentialsModel(BaseModel):
diff --git a/src/middlewared/middlewared/rclone/remote/storjix.py b/src/middlewared/middlewared/rclone/remote/storjix.py
index 201dc3bfedead..fa257b06e6d57 100644
--- a/src/middlewared/middlewared/rclone/remote/storjix.py
+++ b/src/middlewared/middlewared/rclone/remote/storjix.py
@@ -1,5 +1,6 @@
 import errno
 import io
+from urllib.parse import urlparse
 import xml.etree.ElementTree as ET
 
 from aws_requests_auth.aws_auth import AWSRequestsAuth
@@ -32,12 +33,13 @@ class StorjIxRcloneRemote(BaseRcloneRemote):
 
     async def create_bucket(self, credentials, name):
         def create_bucket_sync():
+            provider = credentials["provider"]
             s3_client = boto3.client(
                 "s3",
                 config=botocore.config.Config(user_agent="ix-storj-1"),
-                endpoint_url="https://gateway.storjshare.io",
-                aws_access_key_id=credentials["provider"]["access_key_id"],
-                aws_secret_access_key=credentials["provider"]["secret_access_key"],
+                endpoint_url=provider["endpoint"],
+                aws_access_key_id=provider["access_key_id"],
+                aws_secret_access_key=provider["secret_access_key"],
             )
             # s3 bucket naming rules: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
             try:
@@ -57,13 +59,16 @@ def create_bucket_sync():
 
     async def list_buckets(self, credentials):
         def list_buckets_sync():
-            auth = AWSRequestsAuth(aws_access_key=credentials["provider"]["access_key_id"],
-                                   aws_secret_access_key=credentials["provider"]["secret_access_key"],
-                                   aws_host="gateway.storjshare.io",
+            provider = credentials["provider"]
+            endpoint = provider["endpoint"]
+
+            auth = AWSRequestsAuth(aws_access_key=provider["access_key_id"],
+                                   aws_secret_access_key=provider["secret_access_key"],
+                                   aws_host=urlparse(endpoint).hostname,
                                    aws_region="",
                                    aws_service="s3")
 
-            r = requests.get("https://gateway.storjshare.io/?attribution", auth=auth, timeout=INTERNET_TIMEOUT)
+            r = requests.get(f"{endpoint}?attribution", auth=auth, timeout=INTERNET_TIMEOUT)
             r.raise_for_status()
 
             ns = "{http://s3.amazonaws.com/doc/2006-03-01/}"
@@ -79,7 +84,7 @@ def list_buckets_sync():
         return await self.middleware.run_in_thread(list_buckets_sync)
 
     async def get_credentials_extra(self, credentials):
-        return {"endpoint": "https://gateway.storjshare.io", "provider": "Other"}
+        return {"endpoint": credentials["provider"]["endpoint"], "provider": "Other"}
 
     async def get_task_extra(self, task):
         # Storj recommended these settings
@@ -90,11 +95,9 @@ async def get_task_extra(self, task):
         }
 
     def get_restic_config(self, task):
-        url = "gateway.storjshare.io"
-
+        provider = task["credentials"]["provider"]
         env = {
-            "AWS_ACCESS_KEY_ID": task["credentials"]["provider"]["access_key_id"],
-            "AWS_SECRET_ACCESS_KEY": task["credentials"]["provider"]["secret_access_key"],
+            "AWS_ACCESS_KEY_ID": provider["access_key_id"],
+            "AWS_SECRET_ACCESS_KEY": provider["secret_access_key"],
         }
-
-        return url, env
+        return urlparse(provider["endpoint"]).hostname, env


Reviews:
user: login: themylogin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T16:25:25Z
user: login: themylogin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T16:26:02Z
user: login: creatorcary
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T16:52:00Z
user: login: themylogin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T16:55:59Z
user: login: creatorcary
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T17:04:36Z
user: login: themylogin
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T17:12:11Z

Review comments:
diff_hunk: @@ -57,13 +58,16 @@ def create_bucket_sync():
 
     async def list_buckets(self, credentials):
         def list_buckets_sync():
-            auth = AWSRequestsAuth(aws_access_key=credentials["provider"]["access_key_id"],
-                                   aws_secret_access_key=credentials["provider"]["secret_access_key"],
-                                   aws_host="gateway.storjshare.io",
+            provider = credentials["provider"]
+            endpoint = provider["endpoint"]
+
+            auth = AWSRequestsAuth(aws_access_key=provider["access_key_id"],
+                                   aws_secret_access_key=provider["secret_access_key"],
+                                   aws_host=endpoint.removeprefix("https://").strip("/"),
path: src/middlewared/middlewared/rclone/remote/storjix.py
user: login: themylogin
body: The official way to extract host from URL is to use `urllib.parse.urlparse`
created_at: 2025-05-12T16:25:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -90,11 +94,9 @@ async def get_task_extra(self, task):
         }
 
     def get_restic_config(self, task):
-        url = "gateway.storjshare.io"
-
+        provider = task["credentials"]["provider"]
         env = {
-            "AWS_ACCESS_KEY_ID": task["credentials"]["provider"]["access_key_id"],
-            "AWS_SECRET_ACCESS_KEY": task["credentials"]["provider"]["secret_access_key"],
+            "AWS_ACCESS_KEY_ID": provider["access_key_id"],
+            "AWS_SECRET_ACCESS_KEY": provider["secret_access_key"],
         }
-
-        return url, env
+        return provider["endpoint"].removeprefix("https://").strip("/"), env
path: src/middlewared/middlewared/rclone/remote/storjix.py
user: login: themylogin
body: Same here
created_at: 2025-05-12T16:26:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,13 +58,16 @@ def create_bucket_sync():
 
     async def list_buckets(self, credentials):
         def list_buckets_sync():
-            auth = AWSRequestsAuth(aws_access_key=credentials["provider"]["access_key_id"],
-                                   aws_secret_access_key=credentials["provider"]["secret_access_key"],
-                                   aws_host="gateway.storjshare.io",
+            provider = credentials["provider"]
+            endpoint = provider["endpoint"]
+
+            auth = AWSRequestsAuth(aws_access_key=provider["access_key_id"],
+                                   aws_secret_access_key=provider["secret_access_key"],
+                                   aws_host=endpoint.removeprefix("https://").strip("/"),
path: src/middlewared/middlewared/rclone/remote/storjix.py
user: login: creatorcary
body: Perfect, thank you!
created_at: 2025-05-12T16:52:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,13 +59,16 @@ def create_bucket_sync():
 
     async def list_buckets(self, credentials):
         def list_buckets_sync():
-            auth = AWSRequestsAuth(aws_access_key=credentials["provider"]["access_key_id"],
-                                   aws_secret_access_key=credentials["provider"]["secret_access_key"],
-                                   aws_host="gateway.storjshare.io",
+            provider = credentials["provider"]
+            endpoint = provider["endpoint"]
+
+            auth = AWSRequestsAuth(aws_access_key=provider["access_key_id"],
+                                   aws_secret_access_key=provider["secret_access_key"],
+                                   aws_host=urlparse(endpoint).netloc,
path: src/middlewared/middlewared/rclone/remote/storjix.py
user: login: themylogin
body: `netloc` includes port number. It should be just `hostname`
created_at: 2025-05-12T16:55:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -57,13 +59,16 @@ def create_bucket_sync():
 
     async def list_buckets(self, credentials):
         def list_buckets_sync():
-            auth = AWSRequestsAuth(aws_access_key=credentials["provider"]["access_key_id"],
-                                   aws_secret_access_key=credentials["provider"]["secret_access_key"],
-                                   aws_host="gateway.storjshare.io",
+            provider = credentials["provider"]
+            endpoint = provider["endpoint"]
+
+            auth = AWSRequestsAuth(aws_access_key=provider["access_key_id"],
+                                   aws_secret_access_key=provider["secret_access_key"],
+                                   aws_host=urlparse(endpoint).netloc,
path: src/middlewared/middlewared/rclone/remote/storjix.py
user: login: creatorcary
body: Cool, wasn't aware of the `hostname` property. Granted I don't think a port number should ever be provided in this instance.
created_at: 2025-05-12T17:04:36Z
author_association: CONTRIBUTOR

PR comments:
user: login: bugclerk
created_at: 2025-05-12T15:38:04Z
body: Jira URL: https://ixsystems.atlassian.net/browse/NAS-133835
author_association: CONTRIBUTOR
user: login: creatorcary
created_at: 2025-05-12T19:22:51Z
body: Waiting to see if we can backport this
author_association: CONTRIBUTOR
user: login: themylogin
created_at: 2025-05-12T21:01:59Z
body: @creatorcary if we do, the migration must be put to 24.04 tree
author_association: CONTRIBUTOR
user: login: creatorcary
created_at: 2025-05-13T14:32:35Z
body: backport
author_association: CONTRIBUTOR
user: login: bugclerk
created_at: 2025-05-13T15:24:06Z
body: This PR has been merged and conversations have been locked.
If you would like to discuss more about this issue please use our forums or raise a Jira ticket.

author_association: CONTRIBUTOR