Repository: go-kratos/kratos (go)

Issue:
[Feature] 配置文件增加对环境变量占位符的替换功能
## 背景和动机

我们不希望将一些敏感的内容提交到 git 仓库中，所以这些敏感内容我们会和提交到 git 仓库的配置文件分离。

我们将这些机密内容放在 `dotenv运行时加载到环境变量`  或是 `直接存于环境变量` 中。

除此之外，配置可能因为程序运行环境发生改变，而需要动态的进行配置。

所以我们希望 kratos 能够在配置文件中动态的识别环境变量占位符，并使用环境变量值替换配置的内容。

## 提议的方案

> 修订1：提议方案需根据 @tonybase  [描述建议](https://github.com/go-kratos/kratos/issues/1065#issuecomment-865932445)修正
> 修订2：语义占位符采用 `os.Expand` 解析规则完成，废弃修订1的讨论

提案在配置文件中，通过占位符来替换环境变量中的功能:

提案建议的占位符风格有如下几种：

- `%(VARIABLE_NAME)`  ：传统的环境变量占位符
- `%env(VARIABLE_NAME)%` ： 带有环境变量处理器的占位符

#### 转义字符

双倍的 `%%` 将会被视作转义，从而豁免 `%` 占位符被解析

### 传统的环境变量占位符

传统的占位符替换机制只能够替换环境变量（包含 dotenv 运行时加载）的内容，占位符采用 `%()` 包装变量名：

```
data:
  database:
    driver: mysql
    source: %(DB_USERNAME):%(DB_PASSWORD)@tcp(127.0.0.1:3306)/test
```

### 带有环境变量处理器的占位符

环境变量处理器可以使配置程序具备更高的可扩展性，但相对会引入更高的复杂度。环境变量处理器实现可以被动态扩展。

占位符结构中 process 可形成管道:

```
%env([process:]VARIABLE_NAME)%
%env([process-a:][process-b:]VARIABLE_NAME)%
```

### 默认的处理器

提案中提议了一些默认的处理器方案，除了默认方案外，处理器方案可自由扩展(kratos 提出一种接口规范，使开发者可以扩展处理器)：

#### 读取环境变量

等价于传统的环境变量占位符：

```
data:
  database:
    user: %env(DB_USERNAME)%  # 等价于 %(DB_USERNAME)
```

#### 从文件读取内容的处理程序

从环境变量 `AUTH_FILE` 的值中获取文件存储路径，并读取该文件内容作为值

```
parameters:
    secrets: %env(file:AUTH_FILE)%
```

#### 支持 fallback 的环境变量读取

当指定环境变量读取失败时，可以使用默认环境变量的值替代

```
parameters:
    # 环境变量 PRIVATE_KEY 不存在，则使用 raw_key 值（非环境变量解析）替代
    private_key_a: %env(default:raw_key:PRIVATE_KEY)%

    # 环境变量 PRIVATE_KEY 不是一个有效的文件路径，使用 raw_key 值（非环境变量解析）替代
    private_key_b: %env(default:raw_key:file:PRIVATE_KEY)%
```

## 补充

- yaml 标准本身不支持变量占位符
- 占位符格式应在不同配置格式中都具备良好的可读性

Pull request:
feat(config): support Resolver for config variable placeholders
using go template lib to replace placeholder to environment variable

like this:
```yaml
http:
  server:
    # replace with env var "PORT" with default value 8080 
    port: ${PORT:8080}
    # replace with env var "TIMEOUT" without default value
    timeout: "${TIMEOUT}"
```

please help for code reviewing @ymh199478

thanks in advance :-)

Patch:
diff --git a/README.md b/README.md
index 6163f155..2fcef1af 100644
--- a/README.md
+++ b/README.md
@@ -97,3 +97,13 @@ Thank you for considering contributing to the Kratos framework! The contribution
 ## License
 
 The Kratos framework is open-sourced software licensed under the [MIT license](./LICENSE).
+  
+## Acknowledgments
+  
+The following project had particular influence on kratos's design.
+  
+- [go-kit/kit](https://github.com/go-kit/kit) is a programming toolkit for building microservices in go.
+- [asim/go-micro](https://github.com/asim/go-micro) a distributed systems development framework.
+- [google/go-cloud](https://github.com/google/go-cloud) is go cloud development kit.
+- [tal-tech/go-zero](https://github.com/tal-tech/go-zero) is a web and rpc framework with lots of builtin engineering practices.
+- [beego/beego](https://github.com/beego/beego) is a web framework including RESTful APIs, web apps and backend services.
diff --git a/cmd/kratos/internal/run/run.go b/cmd/kratos/internal/run/run.go
index 37609747..37bcbd5f 100644
--- a/cmd/kratos/internal/run/run.go
+++ b/cmd/kratos/internal/run/run.go
@@ -68,6 +68,7 @@ func Run(cmd *cobra.Command, args []string) {
 }
 
 func findCMD(base string) (string, []string, error) {
+	var root bool
 	next := func(dir string) (string, []string, error) {
 		var (
 			cmdDir  string
@@ -88,6 +89,9 @@ func findCMD(base string) (string, []string, error) {
 				cmdDir = filepath.Join(walkPath, "..")
 				return nil
 			}
+			if info.Name() == "go.mod" {
+				root = true
+			}
 			return nil
 		})
 		return cmdDir, cmdPath, err
@@ -100,6 +104,9 @@ func findCMD(base string) (string, []string, error) {
 		if len(res) > 0 {
 			return cmdDir, res, nil
 		}
+		if root {
+			break
+		}
 		base = filepath.Join(base, "..")
 	}
 	return "", []string{base}, nil
diff --git a/config/config.go b/config/config.go
index ccefd7f4..c6f5677d 100644
--- a/config/config.go
+++ b/config/config.go
@@ -2,12 +2,10 @@ package config
 
 import (
 	"errors"
-	"fmt"
 	"reflect"
 	"sync"
 	"time"
 
-	"github.com/go-kratos/kratos/v2/encoding"
 	"github.com/go-kratos/kratos/v2/log"
 
 	// init encoding
@@ -50,17 +48,9 @@ type config struct {
 // New new a config with options.
 func New(opts ...Option) Config {
 	options := options{
-		logger: log.DefaultLogger,
-		decoder: func(src *KeyValue, target map[string]interface{}) error {
-			if src.Format == "" {
-				target[src.Key] = src.Value
-				return nil
-			}
-			if codec := encoding.GetCodec(src.Format); codec != nil {
-				return codec.Unmarshal(src.Value, &target)
-			}
-			return fmt.Errorf("unsupported key: %s format: %s", src.Key, src.Format)
-		},
+		logger:   log.DefaultLogger,
+		decoder:  defaultDecoder,
+		resolver: defaultResolver,
 	}
 	for _, o := range opts {
 		o(&options)
diff --git a/config/options.go b/config/options.go
index 5bc9ebf0..278698b3 100644
--- a/config/options.go
+++ b/config/options.go
@@ -1,19 +1,28 @@
 package config
 
 import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/go-kratos/kratos/v2/encoding"
 	"github.com/go-kratos/kratos/v2/log"
 )
 
 // Decoder is config decoder.
 type Decoder func(*KeyValue, map[string]interface{}) error
 
+// Resolver resolve placeholder in config.
+type Resolver func(map[string]interface{}) error
+
 // Option is config option.
 type Option func(*options)
 
 type options struct {
-	sources []Source
-	decoder Decoder
-	logger  log.Logger
+	sources  []Source
+	decoder  Decoder
+	resolver Resolver
+	logger   log.Logger
 }
 
 // WithSource with config source.
@@ -30,9 +39,67 @@ func WithDecoder(d Decoder) Option {
 	}
 }
 
+// WithResolver with config resolver.
+func WithResolver(r Resolver) Option {
+	return func(o *options) {
+		o.resolver = r
+	}
+}
+
 // WithLogger with config logger.
 func WithLogger(l log.Logger) Option {
 	return func(o *options) {
 		o.logger = l
 	}
 }
+
+// defaultDecoder decode config from source KeyValue
+// to target map[string]interface{} using src.Format codec.
+func defaultDecoder(src *KeyValue, target map[string]interface{}) error {
+	if src.Format == "" {
+		target[src.Key] = src.Value
+		return nil
+	}
+	if codec := encoding.GetCodec(src.Format); codec != nil {
+		return codec.Unmarshal(src.Value, &target)
+	}
+	return fmt.Errorf("unsupported key: %s format: %s", src.Key, src.Format)
+}
+
+// defaultResolver resolve placeholder in map value,
+// placeholder format in ${key:default} or $key.
+func defaultResolver(input map[string]interface{}) error {
+	mapper := func(name string) string {
+		args := strings.Split(strings.TrimSpace(name), ":")
+		if v, has := readValue(input, args[0]); has {
+			s, _ := v.String()
+			return s
+		} else if len(args) > 1 { // default value
+			return args[1]
+		}
+		return ""
+	}
+
+	var resolve func(map[string]interface{}) error
+	resolve = func(sub map[string]interface{}) error {
+		for k, v := range sub {
+			switch vt := v.(type) {
+			case string:
+				sub[k] = os.Expand(vt, mapper)
+			case map[string]interface{}:
+				if err := resolve(vt); err != nil {
+					return err
+				}
+			case []interface{}:
+				for i, iface := range vt {
+					if s, ok := iface.(string); ok {
+						vt[i] = os.Expand(s, mapper)
+					}
+				}
+				sub[k] = vt
+			}
+		}
+		return nil
+	}
+	return resolve(input)
+}
diff --git a/config/reader.go b/config/reader.go
index 71806a3c..2dc778c3 100644
--- a/config/reader.go
+++ b/config/reader.go
@@ -43,6 +43,9 @@ func (r *reader) Merge(kvs ...*KeyValue) error {
 			return err
 		}
 	}
+	if err := r.opts.resolver(merged); err != nil {
+		return err
+	}
 	r.values = merged
 	return nil
 }
diff --git a/examples/blog/internal/data/data.go b/examples/blog/internal/data/data.go
index 67254f17..0bbaa829 100644
--- a/examples/blog/internal/data/data.go
+++ b/examples/blog/internal/data/data.go
@@ -2,14 +2,20 @@ package data
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/go-kratos/kratos/examples/blog/internal/conf"
 	"github.com/go-kratos/kratos/examples/blog/internal/data/ent"
+
+	"entgo.io/ent/dialect"
+	"entgo.io/ent/dialect/sql"
 	"github.com/go-kratos/kratos/v2/log"
 	"github.com/go-redis/redis/extra/redisotel"
 	"github.com/go-redis/redis/v8"
 	"github.com/google/wire"
-
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	// init mysql driver
 	_ "github.com/go-sql-driver/mysql"
 )
@@ -26,10 +32,24 @@ type Data struct {
 // NewData .
 func NewData(conf *conf.Data, logger log.Logger) (*Data, func(), error) {
 	log := log.NewHelper(logger)
-	client, err := ent.Open(
+	drv, err := sql.Open(
 		conf.Database.Driver,
 		conf.Database.Source,
 	)
+	sqlDrv := dialect.DebugWithContext(drv, func(ctx context.Context, i ...interface{}) {
+		log.WithContext(ctx).Info(i...)
+		tracer := otel.Tracer("ent.")
+		kind := trace.SpanKindServer
+		_, span := tracer.Start(ctx,
+			"Query",
+			trace.WithAttributes(
+				attribute.String("sql", fmt.Sprint(i...)),
+			),
+			trace.WithSpanKind(kind),
+		)
+		span.End()
+	})
+	client := ent.NewClient(ent.Driver(sqlDrv))
 	if err != nil {
 		log.Errorf("failed opening connection to sqlite: %v", err)
 		return nil, nil, err
diff --git a/examples/http/echo/main.go b/examples/http/echo/main.go
new file mode 100644
index 00000000..442645fd
--- /dev/null
+++ b/examples/http/echo/main.go
@@ -0,0 +1,28 @@
+package main
+
+import (
+	"github.com/go-kratos/kratos/v2"
+	"github.com/go-kratos/kratos/v2/transport/http"
+	"github.com/labstack/echo/v4"
+	"log"
+)
+
+func main() {
+	router := echo.New()
+	router.GET("/home", func(ctx echo.Context) error {
+		return ctx.JSON(200,"Hello echo")
+	})
+
+	httpSrv := http.NewServer(http.Address(":8000"))
+	httpSrv.HandlePrefix("/", router)
+
+	app := kratos.New(
+		kratos.Name("echo"),
+		kratos.Server(
+			httpSrv,
+		),
+	)
+	if err := app.Run(); err != nil {
+		log.Fatal(err)
+	}
+}
\ No newline at end of file


Test patch:
diff --git a/config/config_test.go b/config/config_test.go
new file mode 100644
index 00000000..f9bb5a7b
--- /dev/null
+++ b/config/config_test.go
@@ -0,0 +1,141 @@
+package config
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultResolver(t *testing.T) {
+	var (
+		portString = "8080"
+		countInt   = 10
+		enableBool = true
+		rateFloat  = 0.9
+	)
+
+	data := map[string]interface{}{
+		"foo": map[string]interface{}{
+			"bar": map[string]interface{}{
+				"notexist": "${NOTEXIST:100}",
+				"port":     "${PORT:8081}",
+				"count":    "${COUNT:0}",
+				"enable":   "${ENABLE:false}",
+				"rate":     "${RATE}",
+				"empty":    "${EMPTY:foobar}",
+				"array":    []interface{}{"${PORT}", "${NOTEXIST:8081}"},
+				"value1":   "${test.value}",
+				"value2":   "$PORT",
+				"value3":   "$PORT:default",
+			},
+		},
+		"test": map[string]interface{}{
+			"value": "foobar",
+		},
+		"PORT":   "8080",
+		"COUNT":  "10",
+		"ENABLE": "true",
+		"RATE":   "0.9",
+		"EMPTY":  "",
+	}
+
+	tests := []struct {
+		name   string
+		path   string
+		expect interface{}
+	}{
+		{
+			name:   "test not exist int env with default",
+			path:   "foo.bar.notexist",
+			expect: 100,
+		},
+		{
+			name:   "test string with default",
+			path:   "foo.bar.port",
+			expect: portString,
+		},
+		{
+			name:   "test int with default",
+			path:   "foo.bar.count",
+			expect: countInt,
+		},
+		{
+			name:   "test bool with default",
+			path:   "foo.bar.enable",
+			expect: enableBool,
+		},
+		{
+			name:   "test float without default",
+			path:   "foo.bar.rate",
+			expect: rateFloat,
+		},
+		{
+			name:   "test empty value with default",
+			path:   "foo.bar.empty",
+			expect: "",
+		},
+		{
+			name:   "test array",
+			path:   "foo.bar.array",
+			expect: []interface{}{portString, "8081"},
+		},
+		{
+			name:   "test ${test.value}",
+			path:   "foo.bar.value1",
+			expect: "foobar",
+		},
+		{
+			name:   "test $value",
+			path:   "foo.bar.value2",
+			expect: portString,
+		},
+		{
+			name:   "test $value:default",
+			path:   "foo.bar.value3",
+			expect: portString + ":default",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := defaultResolver(data)
+			assert.NoError(t, err)
+			rd := reader{
+				values: data,
+			}
+			if v, ok := rd.Value(test.path); ok {
+				var actual interface{}
+				switch test.expect.(type) {
+				case int:
+					if actual, err = v.Int(); err == nil {
+						assert.Equal(t, test.expect, int(actual.(int64)), "int value should be equal")
+					}
+				case string:
+					if actual, err = v.String(); err == nil {
+						assert.Equal(t, test.expect, actual, "string value should be equal")
+					}
+				case bool:
+					if actual, err = v.Bool(); err == nil {
+						assert.Equal(t, test.expect, actual, "bool value should be equal")
+					}
+				case float64:
+					if actual, err = v.Float(); err == nil {
+						assert.Equal(t, test.expect, actual, "float64 value should be equal")
+					}
+				default:
+					actual = v.Load()
+					if !reflect.DeepEqual(test.expect, actual) {
+						t.Logf("expect: %#v, actural: %#v", test.expect, actual)
+						t.Fail()
+					}
+				}
+				if err != nil {
+					t.Error(err)
+				}
+			} else {
+				t.Error("value path not found")
+			}
+		})
+	}
+}
diff --git a/config/reader_test.go b/config/reader_test.go
index d7c3d179..2cec467a 100644
--- a/config/reader_test.go
+++ b/config/reader_test.go
@@ -20,6 +20,7 @@ func TestReader_Merge(t *testing.T) {
 			}
 			return fmt.Errorf("unsupported key: %s format: %s", kv.Key, kv.Format)
 		},
+		resolver: defaultResolver,
 	}
 	r := newReader(opts)
 	err = r.Merge(&KeyValue{
@@ -55,10 +56,6 @@ func TestReader_Merge(t *testing.T) {
 }
 
 func TestReader_Value(t *testing.T) {
-	var (
-		err error
-		ok  bool
-	)
 	opts := options{
 		decoder: func(kv *KeyValue, v map[string]interface{}) error {
 			if codec := encoding.GetCodec(kv.Format); codec != nil {
@@ -66,42 +63,73 @@ func TestReader_Value(t *testing.T) {
 			}
 			return fmt.Errorf("unsupported key: %s format: %s", kv.Key, kv.Format)
 		},
+		resolver: defaultResolver,
 	}
-	r := newReader(opts)
-	err = r.Merge(&KeyValue{
-		Key:    "b",
-		Value:  []byte(`{"a": {"b": {"X": 1, "Y": "lol", "z": true}}}`),
-		Format: "json",
-	})
-	assert.NoError(t, err)
-	vv, ok := r.Value("a.b.X")
-	assert.True(t, ok)
-	vvv, err := vv.Int()
-	assert.NoError(t, err)
-	assert.Equal(t, int64(1), vvv)
 
-	assert.NoError(t, err)
-	vv, ok = r.Value("a.b.Y")
-	assert.True(t, ok)
-	vvy, err := vv.String()
-	assert.NoError(t, err)
-	assert.Equal(t, "lol", vvy)
+	ymlval := `
+a: 
+  b: 
+    X: 1
+    Y: "lol"
+    z: true
+`
+	tests := []struct {
+		name string
+		kv   KeyValue
+	}{
+		{
+			name: "json value",
+			kv: KeyValue{
+				Key:    "config",
+				Value:  []byte(`{"a": {"b": {"X": 1, "Y": "lol", "z": true}}}`),
+				Format: "json",
+			},
+		},
+		{
+			name: "yaml value",
+			kv: KeyValue{
+				Key:    "config",
+				Value:  []byte(ymlval),
+				Format: "yaml",
+			},
+		},
+	}
 
-	assert.NoError(t, err)
-	vv, ok = r.Value("a.b.z")
-	assert.True(t, ok)
-	vvz, err := vv.Bool()
-	assert.NoError(t, err)
-	assert.Equal(t, true, vvz)
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			r := newReader(opts)
+			err := r.Merge(&test.kv)
+			assert.NoError(t, err)
+			vv, ok := r.Value("a.b.X")
+			assert.True(t, ok)
+			vvv, err := vv.Int()
+			assert.NoError(t, err)
+			assert.Equal(t, int64(1), vvv)
 
-	vv, ok = r.Value("aasasdg=234l.asdfk,")
-	assert.False(t, ok)
+			assert.NoError(t, err)
+			vv, ok = r.Value("a.b.Y")
+			assert.True(t, ok)
+			vvy, err := vv.String()
+			assert.NoError(t, err)
+			assert.Equal(t, "lol", vvy)
 
-	vv, ok = r.Value("aas......asdg=234l.asdfk,")
-	assert.False(t, ok)
+			assert.NoError(t, err)
+			vv, ok = r.Value("a.b.z")
+			assert.True(t, ok)
+			vvz, err := vv.Bool()
+			assert.NoError(t, err)
+			assert.Equal(t, true, vvz)
 
-	vv, ok = r.Value("a.b.Y.")
-	assert.False(t, ok)
+			vv, ok = r.Value("aasasdg=234l.asdfk,")
+			assert.False(t, ok)
+
+			vv, ok = r.Value("aas......asdg=234l.asdfk,")
+			assert.False(t, ok)
+
+			vv, ok = r.Value("a.b.Y.")
+			assert.False(t, ok)
+		})
+	}
 }
 
 func TestReader_Source(t *testing.T) {
@@ -115,6 +143,7 @@ func TestReader_Source(t *testing.T) {
 			}
 			return fmt.Errorf("unsupported key: %s format: %s", kv.Key, kv.Format)
 		},
+		resolver: defaultResolver,
 	}
 	r := newReader(opts)
 	err = r.Merge(&KeyValue{