Repository: treeverse/lakeFS
Pull request #8954: Support MSB storage
Author: login: guy-har
Labels: name: include-changelog
description: PR description should be included in next release changelog
name: minor-change
description: Used for PRs that don't require issue attached
Created: 2025-04-09T10:01:04Z | Merged: 2025-04-22T12:58:15Z

Closes treeverse/lakeFS-enterprise#48

## Change Description
This change provides the lakeFSFS to support storage config list case
This change:
- uses the config endpoint instead of the depricated internal getStorageConfig endpoint
- Uses getStorageConfigList in case it's provided, otherwise fallbacks to storageConfig

### How was this tested
This was tested by running the sonnets test locally with the following information:
1. Use lakeFSFS instead of gateway
2. Use on lakeFS running with MSB without a bc storage 




Diff:
diff --git a/clients/hadoopfs/pom.xml b/clients/hadoopfs/pom.xml
index ce92aace834..2c4c9e928f9 100644
--- a/clients/hadoopfs/pom.xml
+++ b/clients/hadoopfs/pom.xml
@@ -287,7 +287,7 @@ To export to S3:
         <dependency>
             <groupId>io.lakefs</groupId>
             <artifactId>sdk</artifactId>
-            <version>1.18.0</version>
+            <version>1.53.1</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
diff --git a/clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java b/clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java
index 796df83fba6..079ab796dc0 100644
--- a/clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java
+++ b/clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java
@@ -13,6 +13,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.NoSuchElementException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -51,6 +52,7 @@
 import io.lakefs.clients.sdk.model.PathList;
 import io.lakefs.clients.sdk.model.Repository;
 import io.lakefs.clients.sdk.model.StorageConfig;
+import io.lakefs.clients.sdk.model.Config;
 import io.lakefs.storage.CreateOutputStreamParams;
 import io.lakefs.storage.PhysicalAddressTranslator;
 import io.lakefs.storage.PresignedStorageAccessStrategy;
@@ -78,7 +80,7 @@ public class LakeFSFileSystem extends FileSystem {
 
     private Configuration conf;
     private URI uri;
-    private Path workingDirectory = new Path(Constants.SEPARATOR);
+    private Path workingDirectory = new Path(SEPARATOR);
     private ClientFactory clientFactory;
     private LakeFSClient lfsClient;
     private int listAmount;
@@ -115,6 +117,31 @@ public void initialize(URI name, Configuration conf) throws IOException {
             });
     }
 
+    private StorageConfig getStorageConfig(URI uri) throws IOException, ApiException {
+        Path path = new Path(uri);
+        ObjectLocation objectLoc = pathToObjectLocation(path);
+
+        Repository repository = lfsClient.getRepositoriesApi()
+                .getRepository(objectLoc.getRepository())
+                .execute();
+        String storageID = repository.getStorageId();
+        Config cfg = lfsClient.getConfigApi().getConfig().execute();
+        StorageConfig storageConfig = null;
+        if (storageID == null || Objects.equals(storageID, "")) {
+            storageConfig = cfg.getStorageConfig();
+
+        } else {
+            List<StorageConfig> storageConfigList = cfg.getStorageConfigList();
+            if (storageConfigList != null && !storageConfigList.isEmpty()) {
+                storageConfig = storageConfigList.stream()
+                        .filter(sc -> Objects.equals(sc.getBlockstoreId(), storageID))
+                        .findFirst()
+                        .orElseThrow(() -> new IOException("Failed to get lakeFS blockstore configuration"));
+            }
+        }
+        return storageConfig;
+    }
+
     void initializeWithClientFactory(URI name, Configuration conf, ClientFactory clientFactory) throws IOException {
         super.initialize(name, conf);
         this.uri = name;
@@ -136,7 +163,7 @@ void initializeWithClientFactory(URI name, Configuration conf, ClientFactory cli
         } else if (accessMode == AccessMode.SIMPLE) {
             // setup address translator for simple storage access strategy
             try {
-                StorageConfig storageConfig = lfsClient.getInternalApi().getStorageConfig().execute();
+                StorageConfig storageConfig = getStorageConfig(name);
                 physicalAddressTranslator = new PhysicalAddressTranslator(storageConfig.getBlockstoreType(),
                         storageConfig.getBlockstoreNamespaceValidityRegex());
             } catch (ApiException e) {
@@ -181,7 +208,7 @@ public long getDefaultBlockSize(Path path) {
         if (getFSForConfig() != null) {
             return getFSForConfig().getDefaultBlockSize(path);
         }
-        return Constants.DEFAULT_BLOCK_SIZE;
+        return DEFAULT_BLOCK_SIZE;
     }
 
     @Override
@@ -189,7 +216,7 @@ public long getDefaultBlockSize() {
         if (getFSForConfig() != null) {
             return getFSForConfig().getDefaultBlockSize();
         }
-        return Constants.DEFAULT_BLOCK_SIZE;
+        return DEFAULT_BLOCK_SIZE;
     }
 
     @Override
@@ -735,7 +762,7 @@ private void createDirectoryMarker(Path path) throws IOException {
             ObjectLocation objectLoc = pathToObjectLocation(path).toDirectory();
             OutputStream out = storageAccessStrategy.createDataOutputStream(objectLoc, null);
             out.close();
-        } catch (io.lakefs.clients.sdk.ApiException e) {
+        } catch (ApiException e) {
             throw new IOException("createDirectoryMarker: " + e.getResponseBody(), e);
         }
     }
@@ -744,7 +771,7 @@ private void createDirectoryMarker(Path path) throws IOException {
      * Return a file status object that represents the path.
      * @param path to a file or directory
      * @return a LakeFSFileStatus object
-     * @throws java.io.FileNotFoundException when the path does not exist;
+     * @throws FileNotFoundException when the path does not exist;
      *         IOException API call or underlying filesystem exceptions
      */
     @Override
diff --git a/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java b/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java
index 6b41dd1da4b..5075bca3625 100644
--- a/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java
+++ b/clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java
@@ -144,22 +144,23 @@ public void hadoopSetup() throws IOException, URISyntaxException {
 
         System.setProperty("hadoop.home.dir", "/");
 
-        // lakeFSFS initialization requires a blockstore.
-        mockServerClient.when(request()
-                              .withMethod("GET")
-                              .withPath("/config/storage"),
-                              Times.once())
-            .respond(response()
-                     .withStatusCode(200)
-                     .withBody(gson.toJson(new StorageConfig()
-                                           .blockstoreType("s3")
-                                           .blockstoreNamespaceExample("/not/really")
-                                           .blockstoreNamespaceValidityRegex(".*")
-                                           // TODO(ariels): Change for presigned?
-                                           .preSignSupport(false)
-                                           .preSignSupportUi(false)
-                                           .importSupport(false)
-                                           .importValidityRegex(".*"))));
+        mockServerClient.when(
+                request()
+                        .withMethod("GET")
+                        .withPath("/config"),
+                Times.once()
+        ).respond(
+                response()
+                        .withStatusCode(200)
+                        .withBody(gson.toJson(new Config()
+                                .storageConfig(new StorageConfig()
+                                        .blockstoreType("s3")
+                                        .blockstoreNamespaceExample("unused-but-checked")
+                                        .preSignSupport(false)
+                                        .preSignSupportUi(false)
+                                        .importSupport(false)
+                                        .importValidityRegex(".*")
+                                        .blockstoreNamespaceValidityRegex(".*")))));
 
         // Always allow repo "repo" to be found, it's used in all tests.
         mockServerClient.when(request()


Reviews:
user: login: N-o-Z
body: LGTM!
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-10T12:00:32Z
user: login: arielshaqed
body: Cool, thanks!  Only minor changes requested.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T12:42:23Z

Review comments:
diff_hunk: @@ -115,6 +117,31 @@ public void initialize(URI name, Configuration conf) throws IOException {
             });
     }
 
+    private StorageConfig getStorageConfig(URI uri) throws IOException, ApiException {
+        Path path = new Path(uri);
+        ObjectLocation objectLoc = pathToObjectLocation(path);
+
+        Repository repository = lfsClient.getRepositoriesApi()
+                .getRepository(objectLoc.getRepository())
+                .execute();
+        String storageID = repository.getStorageId();
+        Config cfg = lfsClient.getConfigApi().getConfig().execute();
+        StorageConfig storageConfig = new StorageConfig();
+        if (storageID == null || Objects.equals(storageID, "")) {
+            storageConfig = cfg.getStorageConfig();
+
+        } else {
+            List<StorageConfig> storageConfigList = cfg.getStorageConfigList();
+            if (storageConfigList != null && !storageConfigList.isEmpty()) {
+                storageConfig = storageConfigList.stream()
+                        .filter(sc -> Objects.equals(sc.getBlockstoreId(), storageID))
+                        .findFirst()
+                        .orElseThrow(() -> new IOException("Failed to get lakeFS blockstore type"));
path: clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java
user: login: N-o-Z
body: ```suggestion
                        .orElseThrow(() -> new IOException("Failed to get lakeFS blockstore configuration"));
```
created_at: 2025-04-10T08:01:39Z
author_association: MEMBER
diff_hunk: @@ -287,7 +287,7 @@ To export to S3:
         <dependency>
             <groupId>io.lakefs</groupId>
             <artifactId>sdk</artifactId>
-            <version>1.18.0</version>
+            <version>1.53.1</version>
path: clients/hadoopfs/pom.xml
user: login: arielshaqed
body: Non blocking.  As always, I prefer big version bumps to occur in separate PRs.  Otherwise if things break, it is harder to tell if the breakage was due to the version bump or due to the change in functionality.
created_at: 2025-04-10T12:39:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -115,6 +117,31 @@ public void initialize(URI name, Configuration conf) throws IOException {
             });
     }
 
+    private StorageConfig getStorageConfig(URI uri) throws IOException, ApiException {
+        Path path = new Path(uri);
+        ObjectLocation objectLoc = pathToObjectLocation(path);
+
+        Repository repository = lfsClient.getRepositoriesApi()
+                .getRepository(objectLoc.getRepository())
+                .execute();
+        String storageID = repository.getStorageId();
+        Config cfg = lfsClient.getConfigApi().getConfig().execute();
+        StorageConfig storageConfig = new StorageConfig();
path: clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java
user: login: arielshaqed
body: I don't understand - why create a blank StorageConfig?  If there is a bug and nothing is found, do you really want a _blank_ StorageConfig???
```suggestion
        StorageConfig storageConfig;
```
created_at: 2025-04-10T12:41:11Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-04-09T10:05:46Z
body: 
  # E2E Test Results - Quickstart

  <img src="https://svg.test-summary.com/dashboard.svg?p=12&f=0&s=0" alt="12 passed">

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-09T10:05:50Z
body: 
  # E2E Test Results - DynamoDB Local - Local Block Adapter

  <img src="https://svg.test-summary.com/dashboard.svg?p=13&f=0&s=1" alt="13 passed, 1 skipped">

author_association: NONE