Repository: treeverse/lakeFS
Pull request #8831: Add lakectl support for configuration via environment variable
Author: login: Ben-El
Labels: name: include-changelog
description: PR description should be included in next release changelog
Created: 2025-03-20T15:46:06Z | Merged: 2025-03-23T13:26:52Z

Closes #7519
Note - Relevant only for lakectl, not python sdk !

## Change Description

Previously, the config file could only be set via a flag or defaulted to the user's home directory. This change adds support for specifying the config file using the `LAKECTL_CONFIG_FILE` environment variable.

Linked issue #7519: lakectl / Python SDK: Support passing path to config file through environment variable
Would it make sense to allow specifying the path to the `lakectl` config file through an env var 
(e.g., `LAKECTL_CONFIG`) in addition to passing it through a command-line argument?

The current behavior of either passing the path on the command line or overwriting individual settings through env variables is generally useful. However, in the case of config files in non-standard locations, one needs to always take care to pass the path as part of the command line, making it just a bit more cumbersome.

Similarly, the high-level Python SDK could use the same behavior to facilitate config auto-discovery.

Context: During development on our [lakefs-spec](https://github.com/aai-institute/lakefs-spec) package, I frequently find myself switching between multiple lakeFS server instances for testing. Another use case I can imagine is CI environments, where it's usually easy to mount secret credentials as files (however, without being able to control the path - which prevents them from easily being put under `~/.lakectl.yaml`) and have their path exposed as an environment variable.

If that sounds like a useful addition, I can put together a draft PR.

Slack reference: https://lakefs.slack.com/archives/C016726JLJW/p1709023046347619
Linked issue comments: user: login: idanovo
created_at: 2025-03-19T12:01:19Z
body: @nopcoder to decide on ENV var and overriding order (flag vs env var)
author_association: CONTRIBUTOR
user: login: idanovo
created_at: 2025-03-19T12:03:55Z
body: We should also document this somehow as this is not part of lakectl.yaml configuration or a global flag
author_association: CONTRIBUTOR
user: login: nopcoder
created_at: 2025-03-19T12:09:23Z
body: Use `LAKECTL_CONFIG_FILE` to set the configuration file location and ensure that this property does not conflict with future configuration fields (`config_file` or `config.file`)
author_association: CONTRIBUTOR
user: login: idanovo
created_at: 2025-03-23T13:35:23Z
body: @Ben-El Re-opening the issue as it was only implemented for lakectl and not yet for the Python SDK
author_association: CONTRIBUTOR

Diff:
diff --git a/cmd/lakectl/cmd/root.go b/cmd/lakectl/cmd/root.go
index 0273b6c1ccf..5abada751c6 100644
--- a/cmd/lakectl/cmd/root.go
+++ b/cmd/lakectl/cmd/root.go
@@ -598,6 +598,9 @@ func initConfig() {
 	if cfgFile != "" {
 		// Use config file from the flag.
 		viper.SetConfigFile(cfgFile)
+	} else if envCfgFile, _ := os.LookupEnv("LAKECTL_CONFIG_FILE"); envCfgFile != "" {
+		// Use config file from the env variable.
+		viper.SetConfigFile(envCfgFile)
 	} else {
 		// Find home directory.
 		home, err := os.UserHomeDir()
diff --git a/cmd/lakectl/cmd/root_test.go b/cmd/lakectl/cmd/root_test.go
index 93dd43eb9f8..14aead10e74 100644
--- a/cmd/lakectl/cmd/root_test.go
+++ b/cmd/lakectl/cmd/root_test.go
@@ -4,9 +4,12 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/go-openapi/swag"
+	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/treeverse/lakefs/pkg/osinfo"
@@ -66,3 +69,92 @@ func TestLakectlUnixPerm(t *testing.T) {
 		})
 	}
 }
+
+func TestInitConfig_LoadingScenarios(t *testing.T) {
+	type setupFn func(t *testing.T) (cleanup func())
+
+	tests := []struct {
+		name        string
+		setup       setupFn
+		expectedURL string
+	}{
+		{
+			name: "using --config flag",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "config.yaml")
+				content := "server:\n  endpoint_url: \"http://flag-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				cfgFile = tmpFile
+				return func() {
+					cfgFile = ""
+				}
+			},
+			expectedURL: "http://flag-endpoint",
+		},
+		{
+			name: "using LAKECTL_CONFIG_FILE env var",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "env_config.yaml")
+				content := "server:\n  endpoint_url: \"http://env-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("LAKECTL_CONFIG_FILE", tmpFile)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("LAKECTL_CONFIG_FILE")
+				}
+			},
+			expectedURL: "http://env-endpoint",
+		},
+		{
+			name: "using home directory default",
+			setup: func(t *testing.T) func() {
+				homeDir := t.TempDir()
+				tmpFile := filepath.Join(homeDir, ".lakectl.yaml")
+				content := "server:\n  endpoint_url: \"http://home-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("HOME", homeDir)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("HOME")
+				}
+			},
+			expectedURL: "http://home-endpoint",
+		},
+		{
+			name: "both --config flag and env var are set, flag should win",
+			setup: func(t *testing.T) func() {
+				tmpFileFlag := filepath.Join(t.TempDir(), "config_flag.yaml")
+				flagContent := "server:\n  endpoint_url: \"http://from-flag\"\n"
+				require.NoError(t, os.WriteFile(tmpFileFlag, []byte(flagContent), 0644))
+
+				tmpFileEnv := filepath.Join(t.TempDir(), "config_env.yaml")
+				envContent := "server:\n  endpoint_url: \"http://from-env\"\n"
+				require.NoError(t, os.WriteFile(tmpFileEnv, []byte(envContent), 0644))
+
+				cfgFile = tmpFileFlag
+				t.Setenv("LAKECTL_CONFIG_FILE", tmpFileEnv)
+
+				return func() {
+					cfgFile = ""
+					os.Unsetenv("LAKECTL_CONFIG_FILE")
+				}
+			},
+			expectedURL: "http://from-flag",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfgFile = ""
+			viper.Reset()
+
+			cleanup := tt.setup(t)
+			defer cleanup()
+
+			initConfig()
+			require.NoError(t, viper.Unmarshal(&cfg), "Failed to unmarshal config")
+
+			assert.Equal(t, tt.expectedURL, cfg.Server.EndpointURL.String(), "Unexpected endpoint URL")
+		})
+	}
+}


Reviews:
user: login: idanovo
body: 1. Please remove the prints
2. Where are the tests?
3. The CI fails, please check why
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T19:23:30Z
user: login: idanovo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T11:00:16Z
user: login: idanovo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T11:06:24Z
user: login: idanovo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T11:13:35Z
user: login: idanovo
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T11:16:10Z
user: login: idanovo
body: Nice! almost there
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T11:16:50Z
user: login: Ben-El
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-22T19:05:03Z
user: login: idanovo
body: Last comments I hope 🙏 
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-23T09:40:14Z
user: login: idanovo
body: LGTM!
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-23T10:41:05Z

Review comments:
diff_hunk: @@ -2,8 +2,11 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/spf13/viper"
path: cmd/lakectl/cmd/root_test.go
user: login: idanovo
body: [goimports](https://bariseser.medium.com/useful-go-command-1-goimports-7a9eaa7869a3)/go fmt
created_at: 2025-03-22T11:00:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -66,3 +69,92 @@ func TestLakectlUnixPerm(t *testing.T) {
 		})
 	}
 }
+
+func TestInitConfig_LoadingScenarios(t *testing.T) {
+	type setupFn func(t *testing.T) (cleanup func())
+
+	tests := []struct {
+		name           string
+		setup          setupFn
+		expectedSource string
+		expectedURL    string
+	}{
+		{
+			name: "using --config flag",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "config.yaml")
+				content := "server:\n  endpoint_url: \"http://flag-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				cfgFile = tmpFile
+				return func() {
+					cfgFile = ""
+				}
+			},
+			expectedSource: "http://flag-endpoint",
+			expectedURL:    "http://flag-endpoint",
+		},
+		{
+			name: "using LAKECTL_CONFIG_FILE env var",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "env_config.yaml")
+				content := "server:\n  endpoint_url: \"http://env-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("LAKECTL_CONFIG_FILE", tmpFile)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("LAKECTL_CONFIG_FILE")
+				}
+			},
+			expectedSource: "http://env-endpoint",
+			expectedURL:    "http://env-endpoint",
+		},
+		{
+			name: "using home directory default",
+			setup: func(t *testing.T) func() {
+				homeDir := t.TempDir()
+				tmpFile := filepath.Join(homeDir, ".lakectl.yaml")
+				content := "server:\n  endpoint_url: \"http://home-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("HOME", homeDir)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("HOME")
+				}
+			},
+			expectedSource: "http://home-endpoint",
+			expectedURL:    "http://home-endpoint",
+		},
+		{
+			name: "env-only fallback",
+			setup: func(t *testing.T) func() {
+				t.Setenv("LAKECTL_SERVER_ENDPOINT_URL", "http://env-only")
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("LAKECTL_SERVER_ENDPOINT_URL")
+				}
+			},
+			expectedSource: "",
+			expectedURL:    "http://env-only",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfgFile = ""
+			viper.Reset()
+
+			cleanup := tt.setup(t)
+			defer cleanup()
+
+			initConfig()
+			require.NoError(t, viper.Unmarshal(&cfg), "Failed to unmarshal config")
+
+			if tt.expectedSource != "" {
+				assert.Equal(t, tt.expectedSource, cfg.Server.EndpointURL.String(), "Unexpected endpoint URL")
+				assert.Equal(t, tt.expectedSource, cfg.Server.EndpointURL.String(), "Expected URL from file/env")
path: cmd/lakectl/cmd/root_test.go
user: login: idanovo
body: ```suggestion
				assert.Equal(t, tt.expectedSource, cfg.Server.EndpointURL.String(), "Unexpected endpoint URL")
				assert.Equal(t, tt. expectedURL, cfg.Server.EndpointURL.String(), "Expected URL from file/env")
```
created_at: 2025-03-22T11:06:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -66,3 +69,92 @@ func TestLakectlUnixPerm(t *testing.T) {
 		})
 	}
 }
+
+func TestInitConfig_LoadingScenarios(t *testing.T) {
+	type setupFn func(t *testing.T) (cleanup func())
+
+	tests := []struct {
+		name           string
+		setup          setupFn
+		expectedSource string
+		expectedURL    string
+	}{
+		{
+			name: "using --config flag",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "config.yaml")
+				content := "server:\n  endpoint_url: \"http://flag-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				cfgFile = tmpFile
+				return func() {
+					cfgFile = ""
+				}
+			},
+			expectedSource: "http://flag-endpoint",
+			expectedURL:    "http://flag-endpoint",
+		},
+		{
+			name: "using LAKECTL_CONFIG_FILE env var",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "env_config.yaml")
+				content := "server:\n  endpoint_url: \"http://env-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("LAKECTL_CONFIG_FILE", tmpFile)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("LAKECTL_CONFIG_FILE")
+				}
+			},
+			expectedSource: "http://env-endpoint",
+			expectedURL:    "http://env-endpoint",
+		},
+		{
+			name: "using home directory default",
+			setup: func(t *testing.T) func() {
+				homeDir := t.TempDir()
+				tmpFile := filepath.Join(homeDir, ".lakectl.yaml")
+				content := "server:\n  endpoint_url: \"http://home-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("HOME", homeDir)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("HOME")
+				}
+			},
+			expectedSource: "http://home-endpoint",
+			expectedURL:    "http://home-endpoint",
+		},
+		{
+			name: "env-only fallback",
+			setup: func(t *testing.T) func() {
+				t.Setenv("LAKECTL_SERVER_ENDPOINT_URL", "http://env-only")
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("LAKECTL_SERVER_ENDPOINT_URL")
+				}
+			},
+			expectedSource: "",
+			expectedURL:    "http://env-only",
+		},
path: cmd/lakectl/cmd/root_test.go
user: login: idanovo
body: This test func should only test lakectl configuration file loading, no need to test the env var fallback mechanism
created_at: 2025-03-22T11:13:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -66,3 +69,92 @@ func TestLakectlUnixPerm(t *testing.T) {
 		})
 	}
 }
+
+func TestInitConfig_LoadingScenarios(t *testing.T) {
+	type setupFn func(t *testing.T) (cleanup func())
+
+	tests := []struct {
+		name           string
+		setup          setupFn
+		expectedSource string
+		expectedURL    string
+	}{
+		{
+			name: "using --config flag",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "config.yaml")
+				content := "server:\n  endpoint_url: \"http://flag-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				cfgFile = tmpFile
+				return func() {
+					cfgFile = ""
+				}
+			},
+			expectedSource: "http://flag-endpoint",
+			expectedURL:    "http://flag-endpoint",
+		},
+		{
+			name: "using LAKECTL_CONFIG_FILE env var",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "env_config.yaml")
+				content := "server:\n  endpoint_url: \"http://env-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("LAKECTL_CONFIG_FILE", tmpFile)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("LAKECTL_CONFIG_FILE")
+				}
+			},
+			expectedSource: "http://env-endpoint",
+			expectedURL:    "http://env-endpoint",
+		},
path: cmd/lakectl/cmd/root_test.go
user: login: idanovo
body: You are missing a test where both are used- the `--config` flag and the `LAKECTL_CONFIG_FILE`  env var.
We should validate the flag wins.
created_at: 2025-03-22T11:16:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -66,3 +69,92 @@ func TestLakectlUnixPerm(t *testing.T) {
 		})
 	}
 }
+
+func TestInitConfig_LoadingScenarios(t *testing.T) {
+	type setupFn func(t *testing.T) (cleanup func())
+
+	tests := []struct {
+		name           string
+		setup          setupFn
+		expectedSource string
+		expectedURL    string
+	}{
+		{
+			name: "using --config flag",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "config.yaml")
+				content := "server:\n  endpoint_url: \"http://flag-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				cfgFile = tmpFile
+				return func() {
+					cfgFile = ""
+				}
+			},
+			expectedSource: "http://flag-endpoint",
+			expectedURL:    "http://flag-endpoint",
+		},
+		{
+			name: "using LAKECTL_CONFIG_FILE env var",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "env_config.yaml")
+				content := "server:\n  endpoint_url: \"http://env-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("LAKECTL_CONFIG_FILE", tmpFile)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("LAKECTL_CONFIG_FILE")
+				}
+			},
+			expectedSource: "http://env-endpoint",
+			expectedURL:    "http://env-endpoint",
+		},
path: cmd/lakectl/cmd/root_test.go
user: login: Ben-El
body: Added
created_at: 2025-03-22T19:05:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -66,3 +69,102 @@ func TestLakectlUnixPerm(t *testing.T) {
 		})
 	}
 }
+
+func TestInitConfig_LoadingScenarios(t *testing.T) {
+	type setupFn func(t *testing.T) (cleanup func())
+
+	tests := []struct {
+		name           string
+		setup          setupFn
+		expectedSource string
+		expectedURL    string
+	}{
+		{
+			name: "using --config flag",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "config.yaml")
+				content := "server:\n  endpoint_url: \"http://flag-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				cfgFile = tmpFile
+				return func() {
+					cfgFile = ""
+				}
+			},
+			expectedSource: "http://flag-endpoint",
+			expectedURL:    "http://flag-endpoint",
+		},
+		{
+			name: "using LAKECTL_CONFIG_FILE env var",
+			setup: func(t *testing.T) func() {
+				tmpFile := filepath.Join(t.TempDir(), "env_config.yaml")
+				content := "server:\n  endpoint_url: \"http://env-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("LAKECTL_CONFIG_FILE", tmpFile)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("LAKECTL_CONFIG_FILE")
+				}
+			},
+			expectedSource: "http://env-endpoint",
+			expectedURL:    "http://env-endpoint",
+		},
+		{
+			name: "using home directory default",
+			setup: func(t *testing.T) func() {
+				homeDir := t.TempDir()
+				tmpFile := filepath.Join(homeDir, ".lakectl.yaml")
+				content := "server:\n  endpoint_url: \"http://home-endpoint\"\n"
+				require.NoError(t, os.WriteFile(tmpFile, []byte(content), 0644))
+				t.Setenv("HOME", homeDir)
+				cfgFile = ""
+				return func() {
+					os.Unsetenv("HOME")
+				}
+			},
+			expectedSource: "http://home-endpoint",
+			expectedURL:    "http://home-endpoint",
+		},
+		{
+			name: "both --config flag and env var are set, flag should win",
+			setup: func(t *testing.T) func() {
+				tmpFileFlag := filepath.Join(t.TempDir(), "config_flag.yaml")
+				flagContent := "server:\n  endpoint_url: \"http://from-flag\"\n"
+				require.NoError(t, os.WriteFile(tmpFileFlag, []byte(flagContent), 0644))
+
+				tmpFileEnv := filepath.Join(t.TempDir(), "config_env.yaml")
+				envContent := "server:\n  endpoint_url: \"http://from-env\"\n"
+				require.NoError(t, os.WriteFile(tmpFileEnv, []byte(envContent), 0644))
+
+				cfgFile = tmpFileFlag
+				t.Setenv("LAKECTL_CONFIG_FILE", tmpFileEnv)
+
+				return func() {
+					cfgFile = ""
+					os.Unsetenv("LAKECTL_CONFIG_FILE")
+				}
+			},
+			expectedSource: "http://from-flag",
+			expectedURL:    "http://from-flag",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfgFile = ""
+			viper.Reset()
+
+			cleanup := tt.setup(t)
+			defer cleanup()
+
+			initConfig()
+			require.NoError(t, viper.Unmarshal(&cfg), "Failed to unmarshal config")
+
+			if tt.expectedSource != "" {
+				assert.Equal(t, tt.expectedURL, cfg.Server.EndpointURL.String(), "Unexpected endpoint URL")
+				assert.Equal(t, tt.expectedURL, cfg.Server.EndpointURL.String(), "Expected URL from file/env")
path: cmd/lakectl/cmd/root_test.go
user: login: idanovo
body: You have exactly the same check twice
created_at: 2025-03-23T09:33:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -598,6 +598,9 @@ func initConfig() {
 	if cfgFile != "" {
 		// Use config file from the flag.
 		viper.SetConfigFile(cfgFile)
+	} else if envCfgFile, exists := os.LookupEnv("LAKECTL_CONFIG_FILE"); exists && envCfgFile != "" {
path: cmd/lakectl/cmd/root.go
user: login: idanovo
body: ```suggestion
	} else if envCfgFile := os.LookupEnv("LAKECTL_CONFIG_FILE"); envCfgFile != "" {
```
if the file doesn't exist, envCfgFile will be empty anyway
created_at: 2025-03-23T09:39:41Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-20T15:50:32Z
body: 
  # E2E Test Results - Quickstart

  <img src="https://svg.test-summary.com/dashboard.svg?p=11&f=0&s=0" alt="11 passed">

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-20T15:50:57Z
body: 
  # E2E Test Results - DynamoDB Local - Local Block Adapter

  <img src="https://svg.test-summary.com/dashboard.svg?p=14&f=0&s=0" alt="14 passed">

author_association: NONE