Repository: spf13/viper
Pull request #1869 (2024-06-21 12:28:20)

New encoding layer
Expose a new encoding abstraction layer. It allows users to bring their own encoding logic into Viper.

It also allows moving some of the less used formats (HCL, INI, Java properties) out of the core.

github-actions[bot]: 👋 Thanks for contributing to Viper! You are awesome! 🎉

A maintainer will take a look at your pull request shortly. 👀

In the meantime: We are working on **Viper v2** and we would love to hear your thoughts about what you like or don't like about Viper, so we can improve or fix those issues.

⏰ If you have a couple minutes, please take some time and share your thoughts: https://forms.gle/R6faU74qPRPAzchZ9

📣 If you've already given us your feedback, you can still help by spreading the news,
either by sharing the above link or telling people about this on Twitter:

https://twitter.com/sagikazarmark/status/1306904078967074816

**Thank you!** ❤️

sagikazarmark: @skitt @andig could use a pair of extra eyes on this.
andig: > It also allows moving some of the less used formats (HCL, INI, Java properties) out of the core.

@sagikazarmark this still does not remove the go.mod dependencies, or does it?
sagikazarmark: @andig not yet. This is just the first step to still allow using those libraries once they are removed.
sagikazarmark: I think this is ready

Patch:
diff --git a/encoding.go b/encoding.go
new file mode 100644
index 0000000..b9e090f
--- /dev/null
+++ b/encoding.go
@@ -0,0 +1,226 @@
+package viper
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/spf13/viper/internal/encoding/dotenv"
+	"github.com/spf13/viper/internal/encoding/hcl"
+	"github.com/spf13/viper/internal/encoding/ini"
+	"github.com/spf13/viper/internal/encoding/javaproperties"
+	"github.com/spf13/viper/internal/encoding/json"
+	"github.com/spf13/viper/internal/encoding/toml"
+	"github.com/spf13/viper/internal/encoding/yaml"
+)
+
+// Encoder encodes Viper's internal data structures into a byte representation.
+// It's primarily used for encoding a map[string]any into a file format.
+type Encoder interface {
+	Encode(v map[string]any) ([]byte, error)
+}
+
+// Decoder decodes the contents of a byte slice into Viper's internal data structures.
+// It's primarily used for decoding contents of a file into a map[string]any.
+type Decoder interface {
+	Decode(b []byte, v map[string]any) error
+}
+
+// Codec combines [Encoder] and [Decoder] interfaces.
+type Codec interface {
+	Encoder
+	Decoder
+}
+
+// TODO: consider adding specific errors for not found scenarios
+
+// EncoderRegistry returns an [Encoder] for a given format.
+//
+// Format is case-insensitive.
+//
+// [EncoderRegistry] returns an error if no [Encoder] is registered for the format.
+type EncoderRegistry interface {
+	Encoder(format string) (Encoder, error)
+}
+
+// DecoderRegistry returns an [Decoder] for a given format.
+//
+// Format is case-insensitive.
+//
+// [DecoderRegistry] returns an error if no [Decoder] is registered for the format.
+type DecoderRegistry interface {
+	Decoder(format string) (Decoder, error)
+}
+
+// [CodecRegistry] combines [EncoderRegistry] and [DecoderRegistry] interfaces.
+type CodecRegistry interface {
+	EncoderRegistry
+	DecoderRegistry
+}
+
+// WithEncoderRegistry sets a custom [EncoderRegistry].
+func WithEncoderRegistry(r EncoderRegistry) Option {
+	return optionFunc(func(v *Viper) {
+		v.encoderRegistry = r
+	})
+}
+
+// WithDecoderRegistry sets a custom [DecoderRegistry].
+func WithDecoderRegistry(r DecoderRegistry) Option {
+	return optionFunc(func(v *Viper) {
+		v.decoderRegistry = r
+	})
+}
+
+// WithCodecRegistry sets a custom [EncoderRegistry] and [DecoderRegistry].
+func WithCodecRegistry(r CodecRegistry) Option {
+	return optionFunc(func(v *Viper) {
+		v.encoderRegistry = r
+		v.decoderRegistry = r
+	})
+}
+
+type codecRegistry struct {
+	v *Viper
+}
+
+func (r codecRegistry) Encoder(format string) (Encoder, error) {
+	encoder, ok := r.codec(format)
+	if !ok {
+		return nil, errors.New("encoder not found for this format")
+	}
+
+	return encoder, nil
+}
+
+func (r codecRegistry) Decoder(format string) (Decoder, error) {
+	decoder, ok := r.codec(format)
+	if !ok {
+		return nil, errors.New("decoder not found for this format")
+	}
+
+	return decoder, nil
+}
+
+func (r codecRegistry) codec(format string) (Codec, bool) {
+	switch strings.ToLower(format) {
+	case "yaml", "yml":
+		return yaml.Codec{}, true
+
+	case "json":
+		return json.Codec{}, true
+
+	case "toml":
+		return toml.Codec{}, true
+
+	case "hcl", "tfvars":
+		return hcl.Codec{}, true
+
+	case "ini":
+		return ini.Codec{
+			KeyDelimiter: r.v.keyDelim,
+			LoadOptions:  r.v.iniLoadOptions,
+		}, true
+
+	case "properties", "props", "prop": // Note: This breaks writing a properties file.
+		return &javaproperties.Codec{
+			KeyDelimiter: v.keyDelim,
+		}, true
+
+	case "dotenv", "env":
+		return &dotenv.Codec{}, true
+	}
+
+	return nil, false
+}
+
+// DefaultCodecRegistry is a simple implementation of [CodecRegistry] that allows registering custom [Codec]s.
+type DefaultCodecRegistry struct {
+	codecs map[string]Codec
+
+	mu   sync.RWMutex
+	once sync.Once
+}
+
+// NewCodecRegistry returns a new [CodecRegistry], ready to accept custom [Codec]s.
+func NewCodecRegistry() *DefaultCodecRegistry {
+	r := &DefaultCodecRegistry{}
+
+	r.init()
+
+	return r
+}
+
+func (r *DefaultCodecRegistry) init() {
+	r.once.Do(func() {
+		r.codecs = map[string]Codec{}
+	})
+}
+
+// RegisterCodec registers a custom [Codec].
+//
+// Format is case-insensitive.
+func (r *DefaultCodecRegistry) RegisterCodec(format string, codec Codec) error {
+	r.init()
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	r.codecs[strings.ToLower(format)] = codec
+
+	return nil
+}
+
+// Encoder implements the [EncoderRegistry] interface.
+//
+// Format is case-insensitive.
+func (r *DefaultCodecRegistry) Encoder(format string) (Encoder, error) {
+	encoder, ok := r.codec(format)
+	if !ok {
+		return nil, errors.New("encoder not found for this format")
+	}
+
+	return encoder, nil
+}
+
+// Decoder implements the [DecoderRegistry] interface.
+//
+// Format is case-insensitive.
+func (r *DefaultCodecRegistry) Decoder(format string) (Decoder, error) {
+	decoder, ok := r.codec(format)
+	if !ok {
+		return nil, errors.New("decoder not found for this format")
+	}
+
+	return decoder, nil
+}
+
+func (r *DefaultCodecRegistry) codec(format string) (Codec, bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	format = strings.ToLower(format)
+
+	if r.codecs != nil {
+		codec, ok := r.codecs[format]
+		if ok {
+			return codec, true
+		}
+	}
+
+	switch format {
+	case "yaml", "yml":
+		return yaml.Codec{}, true
+
+	case "json":
+		return json.Codec{}, true
+
+	case "toml":
+		return toml.Codec{}, true
+
+	case "dotenv", "env":
+		return &dotenv.Codec{}, true
+	}
+
+	return nil, false
+}
diff --git a/viper.go b/viper.go
index 30ae4aa..4a0873e 100644
--- a/viper.go
+++ b/viper.go
@@ -40,14 +40,7 @@ import (
 	"github.com/spf13/cast"
 	"github.com/spf13/pflag"
 
-	"github.com/spf13/viper/internal/encoding"
-	"github.com/spf13/viper/internal/encoding/dotenv"
-	"github.com/spf13/viper/internal/encoding/hcl"
 	"github.com/spf13/viper/internal/encoding/ini"
-	"github.com/spf13/viper/internal/encoding/javaproperties"
-	"github.com/spf13/viper/internal/encoding/json"
-	"github.com/spf13/viper/internal/encoding/toml"
-	"github.com/spf13/viper/internal/encoding/yaml"
 	"github.com/spf13/viper/internal/features"
 )
 
@@ -191,9 +184,8 @@ type Viper struct {
 
 	logger *slog.Logger
 
-	// TODO: should probably be protected with a mutex
-	encoderRegistry *encoding.EncoderRegistry
-	decoderRegistry *encoding.DecoderRegistry
+	encoderRegistry EncoderRegistry
+	decoderRegistry DecoderRegistry
 
 	experimentalFinder     bool
 	experimentalBindStruct bool
@@ -217,7 +209,10 @@ func New() *Viper {
 	v.typeByDefValue = false
 	v.logger = slog.New(&discardHandler{})
 
-	v.resetEncoding()
+	codecRegistry := codecRegistry{v: v}
+
+	v.encoderRegistry = codecRegistry
+	v.decoderRegistry = codecRegistry
 
 	v.experimentalFinder = features.Finder
 	v.experimentalBindStruct = features.BindStruct
@@ -268,8 +263,6 @@ func NewWithOptions(opts ...Option) *Viper {
 		opt.apply(v)
 	}
 
-	v.resetEncoding()
-
 	return v
 }
 
@@ -278,16 +271,9 @@ func NewWithOptions(opts ...Option) *Viper {
 // Be careful when using this function: subsequent calls may override options you set.
 // It's always better to use a local Viper instance.
 func SetOptions(opts ...Option) {
-	keyDelim := v.keyDelim
-
 	for _, opt := range opts {
 		opt.apply(v)
 	}
-
-	// reset encoding if key delimiter changed
-	if keyDelim != v.keyDelim {
-		v.resetEncoding()
-	}
 }
 
 // Reset is intended for testing, will reset all to default settings.
@@ -300,84 +286,6 @@ func Reset() {
 	resetRemote()
 }
 
-// TODO: make this lazy initialization instead.
-func (v *Viper) resetEncoding() {
-	encoderRegistry := encoding.NewEncoderRegistry()
-	decoderRegistry := encoding.NewDecoderRegistry()
-
-	{
-		codec := yaml.Codec{}
-
-		encoderRegistry.RegisterEncoder("yaml", codec)
-		decoderRegistry.RegisterDecoder("yaml", codec)
-
-		encoderRegistry.RegisterEncoder("yml", codec)
-		decoderRegistry.RegisterDecoder("yml", codec)
-	}
-
-	{
-		codec := json.Codec{}
-
-		encoderRegistry.RegisterEncoder("json", codec)
-		decoderRegistry.RegisterDecoder("json", codec)
-	}
-
-	{
-		codec := toml.Codec{}
-
-		encoderRegistry.RegisterEncoder("toml", codec)
-		decoderRegistry.RegisterDecoder("toml", codec)
-	}
-
-	{
-		codec := hcl.Codec{}
-
-		encoderRegistry.RegisterEncoder("hcl", codec)
-		decoderRegistry.RegisterDecoder("hcl", codec)
-
-		encoderRegistry.RegisterEncoder("tfvars", codec)
-		decoderRegistry.RegisterDecoder("tfvars", codec)
-	}
-
-	{
-		codec := ini.Codec{
-			KeyDelimiter: v.keyDelim,
-			LoadOptions:  v.iniLoadOptions,
-		}
-
-		encoderRegistry.RegisterEncoder("ini", codec)
-		decoderRegistry.RegisterDecoder("ini", codec)
-	}
-
-	{
-		codec := &javaproperties.Codec{
-			KeyDelimiter: v.keyDelim,
-		}
-
-		encoderRegistry.RegisterEncoder("properties", codec)
-		decoderRegistry.RegisterDecoder("properties", codec)
-
-		encoderRegistry.RegisterEncoder("props", codec)
-		decoderRegistry.RegisterDecoder("props", codec)
-
-		encoderRegistry.RegisterEncoder("prop", codec)
-		decoderRegistry.RegisterDecoder("prop", codec)
-	}
-
-	{
-		codec := &dotenv.Codec{}
-
-		encoderRegistry.RegisterEncoder("dotenv", codec)
-		decoderRegistry.RegisterDecoder("dotenv", codec)
-
-		encoderRegistry.RegisterEncoder("env", codec)
-		decoderRegistry.RegisterDecoder("env", codec)
-	}
-
-	v.encoderRegistry = encoderRegistry
-	v.decoderRegistry = decoderRegistry
-}
-
 // SupportedExts are universally supported extensions.
 var SupportedExts = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl", "tfvars", "dotenv", "env", "ini"}
 
@@ -1715,7 +1623,12 @@ func (v *Viper) unmarshalReader(in io.Reader, c map[string]any) error {
 
 	switch format := strings.ToLower(v.getConfigType()); format {
 	case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini", "properties", "props", "prop", "dotenv", "env":
-		err := v.decoderRegistry.Decode(format, buf.Bytes(), c)
+		decoder, err := v.decoderRegistry.Decoder(format)
+		if err != nil {
+			return ConfigParseError{err}
+		}
+
+		err = decoder.Decode(buf.Bytes(), c)
 		if err != nil {
 			return ConfigParseError{err}
 		}
@@ -1730,7 +1643,12 @@ func (v *Viper) marshalWriter(f afero.File, configType string) error {
 	c := v.AllSettings()
 	switch configType {
 	case "yaml", "yml", "json", "toml", "hcl", "tfvars", "ini", "prop", "props", "properties", "dotenv", "env":
-		b, err := v.encoderRegistry.Encode(configType, c)
+		encoder, err := v.encoderRegistry.Encoder(configType)
+		if err != nil {
+			return ConfigMarshalError{err}
+		}
+
+		b, err := encoder.Encode(c)
 		if err != nil {
 			return ConfigMarshalError{err}
 		}


Test patch:
diff --git a/encoding_test.go b/encoding_test.go
new file mode 100644
index 0000000..3e67a30
--- /dev/null
+++ b/encoding_test.go
@@ -0,0 +1,105 @@
+package viper
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type codec struct{}
+
+func (codec) Encode(_ map[string]any) ([]byte, error) {
+	return nil, nil
+}
+
+func (codec) Decode(_ []byte, _ map[string]any) error {
+	return nil
+}
+
+func TestDefaultCodecRegistry(t *testing.T) {
+	t.Run("OK", func(t *testing.T) {
+		registry := NewCodecRegistry()
+
+		c := codec{}
+
+		err := registry.RegisterCodec("myformat", c)
+		require.NoError(t, err)
+
+		encoder, err := registry.Encoder("myformat")
+		require.NoError(t, err)
+
+		assert.Equal(t, c, encoder)
+
+		decoder, err := registry.Decoder("myformat")
+		require.NoError(t, err)
+
+		assert.Equal(t, c, decoder)
+	})
+
+	t.Run("CodecNotFound", func(t *testing.T) {
+		registry := NewCodecRegistry()
+
+		_, err := registry.Encoder("myformat")
+		require.Error(t, err)
+
+		_, err = registry.Decoder("myformat")
+		require.Error(t, err)
+	})
+
+	t.Run("FormatIsCaseInsensitive", func(t *testing.T) {
+		registry := NewCodecRegistry()
+
+		c := codec{}
+
+		err := registry.RegisterCodec("MYFORMAT", c)
+		require.NoError(t, err)
+
+		{
+			encoder, err := registry.Encoder("myformat")
+			require.NoError(t, err)
+
+			assert.Equal(t, c, encoder)
+		}
+
+		{
+			encoder, err := registry.Encoder("MYFORMAT")
+			require.NoError(t, err)
+
+			assert.Equal(t, c, encoder)
+		}
+
+		{
+			decoder, err := registry.Decoder("myformat")
+			require.NoError(t, err)
+
+			assert.Equal(t, c, decoder)
+		}
+
+		{
+			decoder, err := registry.Decoder("MYFORMAT")
+			require.NoError(t, err)
+
+			assert.Equal(t, c, decoder)
+		}
+	})
+
+	t.Run("OverrideDefault", func(t *testing.T) {
+		registry := NewCodecRegistry()
+
+		c := codec{}
+
+		err := registry.RegisterCodec("yaml", c)
+		require.NoError(t, err)
+
+		encoder, err := registry.Encoder("yaml")
+		require.NoError(t, err)
+
+		assert.Equal(t, c, encoder)
+
+		decoder, err := registry.Decoder("yaml")
+		require.NoError(t, err)
+
+		assert.Equal(t, c, decoder)
+	})
+}
diff --git a/viper_test.go b/viper_test.go
index 2edc83f..05ca0ce 100644
--- a/viper_test.go
+++ b/viper_test.go
@@ -1849,11 +1849,11 @@ var jsonWriteExpected = []byte(`{
   "type": "donut"
 }`)
 
-var propertiesWriteExpected = []byte(`p_id = 0001
-p_type = donut
+var propertiesWriteExpected = []byte(`p_batters.batter.type = Regular
+p_id = 0001
 p_name = Cake
 p_ppu = 0.55
-p_batters.batter.type = Regular
+p_type = donut
 `)
 
 // var yamlWriteExpected = []byte(`age: 35