Repository: TBD54566975/ssi-service (go)

Issue:
[Bug] ServiceKey is overwritten on every restart when MasterKeyPassword is used
**Describe the bug**
We generate a [ServiceKey](https://github.com/TBD54566975/ssi-service/blob/main/pkg/service/keystore/storage.go#L108) on every call to [NewEncryption](https://github.com/TBD54566975/ssi-service/blob/main/pkg/service/keystore/storage.go#L99). The result is that encryption is broken across deployments/pods.

**To Reproduce**
1. Run ssi-service with more than one replica. 
2. Create a key with replica 1. 
3. Read the key with replica 2.
4. Note that the previous step fails with an error.

**Expected behavior**
There should be no error.



Pull request description:
Only use a single encryption key across instances
# Overview
Fixes #560, where multiple instances of the service used different encryption keys.

# Description
This is fixing the bug. 

# How Has This Been Tested?
I tested this by restarting the ssi-service docker container, and checking that the serviceKey didn't change. 


Patch:
diff --git a/pkg/service/keystore/service.go b/pkg/service/keystore/service.go
index 9e1b772..dacc5b2 100644
--- a/pkg/service/keystore/service.go
+++ b/pkg/service/keystore/service.go
@@ -50,12 +50,16 @@ func (s Service) Config() config.KeyStoreServiceConfig {
 }
 
 func NewKeyStoreService(config config.KeyStoreServiceConfig, s storage.ServiceStorage) (*Service, error) {
-	return NewKeyStoreServiceFactory(config, s)(s)
+	if err := EnsureServiceKeyExists(config, s); err != nil {
+		return nil, sdkutil.LoggingErrorMsg(err, "initializing keystore")
+	}
+	factory := NewKeyStoreServiceFactory(config, s)
+	return factory(s)
 }
 
 func NewKeyStoreServiceFactory(config config.KeyStoreServiceConfig, s storage.ServiceStorage) ServiceFactory {
 	return func(tx storage.Tx) (*Service, error) {
-		encrypter, decrypter, err := NewEncryption(s, tx, config)
+		encrypter, decrypter, err := newEncryption(s, config)
 		if err != nil {
 			return nil, errors.Wrap(err, "creating new encryption")
 		}
diff --git a/pkg/service/keystore/storage.go b/pkg/service/keystore/storage.go
index caeac04..4c1eaba 100644
--- a/pkg/service/keystore/storage.go
+++ b/pkg/service/keystore/storage.go
@@ -104,27 +104,56 @@ const (
 	awsKMSScheme = "aws-kms"
 )
 
-func NewEncryption(db storage.ServiceStorage, tx storage.Tx, cfg config.KeyStoreServiceConfig) (Encrypter, Decrypter, error) {
+// EnsureServiceKeyExists makes sure that the service key that will be used for encryption exists. This function is
+// idempotent, so that multiple instances of ssi-service can call it on boot.
+func EnsureServiceKeyExists(config config.KeyStoreServiceConfig, provider storage.ServiceStorage) error {
+	if config.MasterKeyURI != "" {
+		return nil
+	}
+
+	watchKeys := []storage.WatchKey{{
+		Namespace: namespace,
+		Key:       skKey,
+	}}
+
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	if len(cfg.MasterKeyURI) != 0 {
-		return NewExternalEncrypter(ctx, cfg)
-	}
-
-	// First, generate a service key
-	serviceKey, serviceKeySalt, err := GenerateServiceKey(cfg.MasterKeyPassword)
+	_, err := provider.Execute(ctx, func(ctx context.Context, tx storage.Tx) (any, error) {
+		// Create the key only if it doesn't already exist.
+		gotKey, err := getServiceKey(ctx, provider)
+		if gotKey == nil && err.Error() == keyNotFoundErrMsg {
+			serviceKey, serviceKeySalt, err := GenerateServiceKey(config.MasterKeyPassword)
+			if err != nil {
+				return nil, errors.Wrap(err, "generating service key")
+			}
+
+			key := ServiceKey{
+				Base58Key:  serviceKey,
+				Base58Salt: serviceKeySalt,
+			}
+			if err := storeServiceKey(ctx, tx, key); err != nil {
+				return nil, err
+			}
+			return nil, nil
+		}
+		return nil, err
+	}, watchKeys)
 	if err != nil {
-		return nil, nil, sdkutil.LoggingErrorMsg(err, "generating service key")
+		return err
 	}
+	return nil
+}
 
-	key := ServiceKey{
-		Base58Key:  serviceKey,
-		Base58Salt: serviceKeySalt,
-	}
-	if err := storeServiceKey(ctx, tx, key); err != nil {
-		return nil, nil, err
+// newEncryption creates a pair of Encrypter and Decrypter. The service key must have been created before this function
+// is called. EnsureServiceKeyExists can be used to make sure the service key exists.
+func newEncryption(db storage.ServiceStorage, cfg config.KeyStoreServiceConfig) (Encrypter, Decrypter, error) {
+	if len(cfg.MasterKeyURI) != 0 {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		return NewExternalEncrypter(ctx, cfg)
 	}
+
 	return &encrypter{db}, &decrypter{db}, nil
 }
 
diff --git a/pkg/service/service.go b/pkg/service/service.go
index 3a851b1..13d286b 100644
--- a/pkg/service/service.go
+++ b/pkg/service/service.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	sdkutil "github.com/TBD54566975/ssi-sdk/util"
-
 	"github.com/tbd54566975/ssi-service/config"
 	"github.com/tbd54566975/ssi-service/pkg/service/credential"
 	"github.com/tbd54566975/ssi-service/pkg/service/did"
@@ -90,7 +89,13 @@ func instantiateServices(config config.ServicesConfig) (*SSIService, error) {
 		return nil, sdkutil.LoggingErrorMsg(err, "could not instantiate the webhook service")
 	}
 
+	if err := keystore.EnsureServiceKeyExists(config.KeyStoreConfig, storageProvider); err != nil {
+		return nil, sdkutil.LoggingErrorMsg(err, "could not ensure the service key exists")
+	}
 	keyStoreServiceFactory := keystore.NewKeyStoreServiceFactory(config.KeyStoreConfig, storageProvider)
+	if err != nil {
+		return nil, sdkutil.LoggingErrorMsg(err, "could not instantiate the keystore service factory")
+	}
 
 	keyStoreService, err := keyStoreServiceFactory(storageProvider)
 	if err != nil {


Test patch:
diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go
index 1feee7d..3aac072 100644
--- a/pkg/server/server_test.go
+++ b/pkg/server/server_test.go
@@ -230,6 +230,7 @@ func testKeyStoreService(t *testing.T, db storage.ServiceStorage) (*keystore.Ser
 	}
 
 	// create a keystore service
+	require.NoError(t, keystore.EnsureServiceKeyExists(serviceConfig, db))
 	factory := keystore.NewKeyStoreServiceFactory(serviceConfig, db)
 	keystoreService, err := factory(db)
 	require.NoError(t, err)