Repository: nats-io/nats-server
Pull request #3371 (2022-08-16 17:15:29)

[ADDED] Support for AES-GCM as a cipher along with ChaChaPoly. 
Also supports upgrade to different cipher for existing store assets.

Signed-off-by: Derek Collison <derek@nats.io>

/cc @nats-io/core



Patch:
diff --git a/dependencies.md b/dependencies.md
index 4df425b3..294ee8c2 100644
--- a/dependencies.md
+++ b/dependencies.md
@@ -6,15 +6,12 @@ This file lists the dependencies used in this repository.
 |-|-|
 | Go | BSD 3-Clause "New" or "Revised" License |
 | github.com/nats-io/nats-server/v2 | Apache License 2.0 |
-| github.com/golang/protobuf v1.4.2 | BSD 3-Clause "New" or "Revised" License |
-| github.com/nats-io/jwt/v2 v2.3.0 | Apache License 2.0 |
-| github.com/nats-io/nats.go v1.16.1-0.20220810192301-fb5ca2cbc995 | Apache License 2.0 |
-| github.com/nats-io/nkeys v0.3.0 | Apache License 2.0 |
+| github.com/golang/protobuf v1.3.5 |  |
+| github.com/nats-io/jwt/v2 v2.0.0-20201015190852-e11ce317263c | Apache License 2.0 |
+| github.com/nats-io/nats.go v1.10.1-0.20201021145452-94be476ad6e0 | Apache License 2.0 |
+| github.com/nats-io/nkeys v0.2.0 | Apache License 2.0 |
 | github.com/nats-io/nuid v1.0.1 | Apache License 2.0 |
-| golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e | BSD 3-Clause "New" or "Revised" License |
-| golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a | BSD 3-Clause "New" or "Revised" License |
-| golang.org/x/time v0.0.0-20220411224347-583f2d630306 | BSD 3-Clause "New" or "Revised" License |
-| google.golang.org/protobuf v1.23.0 | BSD 3-Clause "New" or "Revised" License |
-| github.com/klauspost/compress v1.15.9 | Apache License 2.0 |
-| github.com/minio/highwayhash v1.0.2 | Apache License 2.0 |
-| go.uber.org/automaxprocs v1.5.1 | MIT License |
+| golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 | BSD 3-Clause "New" or "Revised" License |
+| golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 | BSD 3-Clause "New" or "Revised" License |
+| golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 | BSD 3-Clause "New" or "Revised" License |
+| github.com/minio/highwayhash v1.0.0 | Apache License 2.0 |
diff --git a/server/filestore.go b/server/filestore.go
index 5a745415..64c2db53 100644
--- a/server/filestore.go
+++ b/server/filestore.go
@@ -16,6 +16,7 @@ package server
 import (
 	"archive/tar"
 	"bytes"
+	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
 	"crypto/sha256"
@@ -55,6 +56,8 @@ type FileStoreConfig struct {
 	SyncInterval time.Duration
 	// AsyncFlush allows async flush to batch write operations.
 	AsyncFlush bool
+	// Cipher is the cipher to use when encrypting.
+	Cipher StoreCipher
 }
 
 // FileStreamInfo allows us to remember created time.
@@ -63,6 +66,24 @@ type FileStreamInfo struct {
 	StreamConfig
 }
 
+type StoreCipher int
+
+const (
+	ChaCha StoreCipher = iota
+	AES
+)
+
+func (cipher StoreCipher) String() string {
+	switch cipher {
+	case ChaCha:
+		return "ChaCha20-Poly1305"
+	case AES:
+		return "AES-GCM"
+	default:
+		return "Unknown StoreCipher"
+	}
+}
+
 // File ConsumerInfo is used for creating consumer stores.
 type FileConsumerInfo struct {
 	Created time.Time
@@ -113,7 +134,7 @@ type msgBlock struct {
 	mu      sync.RWMutex
 	fs      *fileStore
 	aek     cipher.AEAD
-	bek     *chacha20.Cipher
+	bek     cipher.Stream
 	seed    []byte
 	nonce   []byte
 	mfn     string
@@ -215,8 +236,8 @@ const (
 	JetStreamMetaFileKey = "meta.key"
 
 	// AEK key sizes
-	metaKeySize = 72
-	blkKeySize  = 72
+	minMetaKeySize = 64
+	minBlkKeySize  = 64
 
 	// Default stream block size.
 	defaultLargeBlockSize = 8 * 1024 * 1024 // 8MB
@@ -443,8 +464,23 @@ func dynBlkSize(retention RetentionPolicy, maxBytes int64) uint64 {
 	}
 }
 
+func genEncryptionKey(sc StoreCipher, seed []byte) (ek cipher.AEAD, err error) {
+	if sc == ChaCha {
+		ek, err = chacha20poly1305.NewX(seed)
+	} else if sc == AES {
+		block, e := aes.NewCipher(seed)
+		if e != nil {
+			return nil, err
+		}
+		ek, err = cipher.NewGCMWithNonceSize(block, block.BlockSize())
+	} else {
+		err = errUnknownCipher
+	}
+	return ek, err
+}
+
 // Generate an asset encryption key from the context and server PRF.
-func (fs *fileStore) genEncryptionKeys(context string) (aek cipher.AEAD, bek *chacha20.Cipher, seed, encrypted []byte, err error) {
+func (fs *fileStore) genEncryptionKeys(context string) (aek cipher.AEAD, bek cipher.Stream, seed, encrypted []byte, err error) {
 	if fs.prf == nil {
 		return nil, nil, nil, nil, errNoEncryption
 	}
@@ -453,16 +489,22 @@ func (fs *fileStore) genEncryptionKeys(context string) (aek cipher.AEAD, bek *ch
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
-	kek, err := chacha20poly1305.NewX(rb)
+
+	sc := fs.fcfg.Cipher
+
+	kek, err := genEncryptionKey(sc, rb)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
 	// Generate random asset encryption key seed.
-	seed = make([]byte, 32)
-	if n, err := rand.Read(seed); err != nil || n != 32 {
+
+	const seedSize = 32
+	seed = make([]byte, seedSize)
+	if n, err := rand.Read(seed); err != nil || n != seedSize {
 		return nil, nil, nil, nil, err
 	}
-	aek, err = chacha20poly1305.NewX(seed)
+
+	aek, err = genEncryptionKey(sc, seed)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
@@ -470,7 +512,8 @@ func (fs *fileStore) genEncryptionKeys(context string) (aek cipher.AEAD, bek *ch
 	// Generate our nonce. Use same buffer to hold encrypted seed.
 	nonce := make([]byte, kek.NonceSize(), kek.NonceSize()+len(seed)+kek.Overhead())
 	mrand.Read(nonce)
-	bek, err = chacha20.NewUnauthenticatedCipher(seed[:], nonce)
+
+	bek, err = genBlockEncryptionKey(sc, seed[:], nonce)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
@@ -478,6 +521,20 @@ func (fs *fileStore) genEncryptionKeys(context string) (aek cipher.AEAD, bek *ch
 	return aek, bek, seed, kek.Seal(nonce, nonce, seed, nil), nil
 }
 
+// Will generate the block encryption key.
+func genBlockEncryptionKey(sc StoreCipher, seed, nonce []byte) (cipher.Stream, error) {
+	if sc == ChaCha {
+		return chacha20.NewUnauthenticatedCipher(seed, nonce)
+	} else if sc == AES {
+		block, err := aes.NewCipher(seed)
+		if err != nil {
+			return nil, err
+		}
+		return cipher.NewCTR(block, nonce), nil
+	}
+	return nil, errUnknownCipher
+}
+
 // Write out meta and the checksum.
 // Lock should be held.
 func (fs *fileStore) writeStreamMeta() error {
@@ -486,7 +543,6 @@ func (fs *fileStore) writeStreamMeta() error {
 		if err != nil {
 			return err
 		}
-		fs.aek = key
 		keyFile := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFileKey)
 		if _, err := os.Stat(keyFile); err != nil && !os.IsNotExist(err) {
 			return err
@@ -494,6 +550,8 @@ func (fs *fileStore) writeStreamMeta() error {
 		if err := os.WriteFile(keyFile, encrypted, defaultFilePerms); err != nil {
 			return err
 		}
+		// Set our aek.
+		fs.aek = key
 	}
 
 	meta := filepath.Join(fs.fcfg.StoreDir, JetStreamMetaFile)
@@ -611,7 +669,7 @@ func (fs *fileStore) recoverMsgBlock(fi os.FileInfo, index uint32) (*msgBlock, e
 			}
 			createdKeys = true
 		} else {
-			if len(ekey) != blkKeySize {
+			if len(ekey) < minBlkKeySize {
 				return nil, errBadKeySize
 			}
 			// Recover key encryption key.
@@ -619,20 +677,27 @@ func (fs *fileStore) recoverMsgBlock(fi os.FileInfo, index uint32) (*msgBlock, e
 			if err != nil {
 				return nil, err
 			}
-			kek, err := chacha20poly1305.NewX(rb)
+
+			sc := fs.fcfg.Cipher
+			kek, err := genEncryptionKey(sc, rb)
 			if err != nil {
 				return nil, err
 			}
 			ns := kek.NonceSize()
 			seed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil)
 			if err != nil {
-				return nil, err
+				// We may be here on a cipher conversion, so attempt to convert.
+				if err = mb.convertCipher(); err != nil {
+					return nil, err
+				}
+			} else {
+				mb.seed, mb.nonce = seed, ekey[:ns]
 			}
-			mb.seed, mb.nonce = seed, ekey[:ns]
-			if mb.aek, err = chacha20poly1305.NewX(seed); err != nil {
+			mb.aek, err = genEncryptionKey(sc, mb.seed)
+			if err != nil {
 				return nil, err
 			}
-			if mb.bek, err = chacha20.NewUnauthenticatedCipher(seed, ekey[:ns]); err != nil {
+			if mb.bek, err = genBlockEncryptionKey(sc, mb.seed, mb.nonce); err != nil {
 				return nil, err
 			}
 		}
@@ -743,6 +808,75 @@ func (fs *fileStore) rebuildStateLocked(ld *LostStreamData) {
 	}
 }
 
+// Attempt to convert the cipher used for this message block.
+func (mb *msgBlock) convertCipher() error {
+	fs := mb.fs
+	sc := fs.fcfg.Cipher
+
+	var osc StoreCipher
+	switch sc {
+	case ChaCha:
+		osc = AES
+	case AES:
+		osc = ChaCha
+	}
+
+	mdir := filepath.Join(fs.fcfg.StoreDir, msgDir)
+	ekey, err := os.ReadFile(filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index)))
+	if err != nil {
+		return err
+	}
+	if len(ekey) < minBlkKeySize {
+		return errBadKeySize
+	}
+	// Recover key encryption key.
+	rb, err := fs.prf([]byte(fmt.Sprintf("%s:%d", fs.cfg.Name, mb.index)))
+	if err != nil {
+		return err
+	}
+	kek, err := genEncryptionKey(osc, rb)
+	if err != nil {
+		return err
+	}
+	ns := kek.NonceSize()
+	seed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil)
+	if err != nil {
+		return err
+	}
+	nonce := ekey[:ns]
+
+	bek, err := genBlockEncryptionKey(osc, seed, nonce)
+	if err != nil {
+		return err
+	}
+
+	buf, _ := mb.loadBlock(nil)
+	bek.XORKeyStream(buf, buf)
+	// Make sure we can parse with old cipher and key file.
+	if err = mb.indexCacheBuf(buf); err != nil {
+		return err
+	}
+	// Reset the cache since we just read everything in.
+	mb.cache = nil
+
+	// Generate new keys based on our
+	if err := fs.genEncryptionKeysForBlock(mb); err != nil {
+		// Put the old keyfile back.
+		keyFile := filepath.Join(mdir, fmt.Sprintf(keyScan, mb.index))
+		os.WriteFile(keyFile, ekey, defaultFilePerms)
+		return err
+	}
+	mb.bek.XORKeyStream(buf, buf)
+	if err := os.WriteFile(mb.mfn, buf, defaultFilePerms); err != nil {
+		return err
+	}
+	// If we are here we want to delete other meta, e.g. idx, fss.
+	os.Remove(mb.ifn)
+	os.Remove(mb.sfn)
+
+	return nil
+}
+
 // Convert a plaintext block to encrypted.
 func (mb *msgBlock) convertToEncrypted() error {
 	if mb.bek == nil {
@@ -797,7 +931,7 @@ func (mb *msgBlock) rebuildStateLocked() (*LostStreamData, error) {
 	// Check if we need to decrypt.
 	if mb.bek != nil && len(buf) > 0 {
 		// Recreate to reset counter.
-		mb.bek, err = chacha20.NewUnauthenticatedCipher(mb.seed, mb.nonce)
+		mb.bek, err = genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce)
 		if err != nil {
 			return nil, err
 		}
@@ -2329,7 +2463,7 @@ func (mb *msgBlock) compact() {
 	// Check for encryption.
 	if mb.bek != nil && len(nbuf) > 0 {
 		// Recreate to reset counter.
-		rbek, err := chacha20.NewUnauthenticatedCipher(mb.seed, mb.nonce)
+		rbek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce)
 		if err != nil {
 			return
 		}
@@ -3584,7 +3718,7 @@ checkCache:
 
 	// Check if we need to decrypt.
 	if mb.bek != nil && len(buf) > 0 {
-		bek, err := chacha20.NewUnauthenticatedCipher(mb.seed, mb.nonce)
+		bek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce)
 		if err != nil {
 			return err
 		}
@@ -3635,18 +3769,19 @@ func (mb *msgBlock) fetchMsg(seq uint64, sm *StoreMsg) (*StoreMsg, bool, error)
 }
 
 var (
-	errNoCache      = errors.New("no message cache")
-	errBadMsg       = errors.New("malformed or corrupt message")
-	errDeletedMsg   = errors.New("deleted message")
-	errPartialCache = errors.New("partial cache")
-	errNoPending    = errors.New("message block does not have pending data")
-	errNotReadable  = errors.New("storage directory not readable")
-	errCorruptState = errors.New("corrupt state file")
-	errPendingData  = errors.New("pending data still present")
-	errNoEncryption = errors.New("encryption not enabled")
-	errBadKeySize   = errors.New("encryption bad key size")
-	errNoMsgBlk     = errors.New("no message block")
-	errMsgBlkTooBig = errors.New("message block size exceeded int capacity")
+	errNoCache       = errors.New("no message cache")
+	errBadMsg        = errors.New("malformed or corrupt message")
+	errDeletedMsg    = errors.New("deleted message")
+	errPartialCache  = errors.New("partial cache")
+	errNoPending     = errors.New("message block does not have pending data")
+	errNotReadable   = errors.New("storage directory not readable")
+	errCorruptState  = errors.New("corrupt state file")
+	errPendingData   = errors.New("pending data still present")
+	errNoEncryption  = errors.New("encryption not enabled")
+	errBadKeySize    = errors.New("encryption bad key size")
+	errNoMsgBlk      = errors.New("no message block")
+	errMsgBlkTooBig  = errors.New("message block size exceeded int capacity")
+	errUnknownCipher = errors.New("unknown cipher")
 )
 
 // Used for marking messages that have had their checksums checked.
@@ -4638,7 +4773,7 @@ func (fs *fileStore) Compact(seq uint64) (uint64, error) {
 			// Check for encryption.
 			if smb.bek != nil && len(nbuf) > 0 {
 				// Recreate to reset counter.
-				bek, err := chacha20.NewUnauthenticatedCipher(smb.seed, smb.nonce)
+				bek, err := genBlockEncryptionKey(smb.fs.fcfg.Cipher, smb.seed, smb.nonce)
 				if err != nil {
 					goto SKIP
 				}
@@ -4839,7 +4974,7 @@ func (mb *msgBlock) closeAndKeepIndex() {
 
 	// If we are encrypted we should reset our bek counter.
 	if mb.bek != nil {
-		if bek, err := chacha20.NewUnauthenticatedCipher(mb.seed, mb.nonce); err == nil {
+		if bek, err := genBlockEncryptionKey(mb.fs.fcfg.Cipher, mb.seed, mb.nonce); err == nil {
 			mb.bek = bek
 		}
 	}
@@ -5388,7 +5523,7 @@ func (fs *fileStore) streamSnapshot(w io.WriteCloser, state *StreamState, includ
 		}
 		// Check for encryption.
 		if mb.bek != nil && len(bbuf) > 0 {
-			rbek, err := chacha20.NewUnauthenticatedCipher(mb.seed, mb.nonce)
+			rbek, err := genBlockEncryptionKey(fs.fcfg.Cipher, mb.seed, mb.nonce)
 			if err != nil {
 				mb.mu.Unlock()
 				writeErr(fmt.Sprintf("Could not create encryption key for message block [%d]: %v", mb.index, err))
@@ -5446,7 +5581,7 @@ func (fs *fileStore) streamSnapshot(w io.WriteCloser, state *StreamState, includ
 			writeErr(fmt.Sprintf("Could not encode consumer state for %q: %v", o.name, err))
 			return
 		}
-		odirPre := consumerDir + "/" + o.name
+		odirPre := filepath.Join(consumerDir, o.name)
 		o.mu.Unlock()
 
 		// Write all the consumer files.
@@ -5571,21 +5706,32 @@ func (fs *fileStore) ConsumerStore(name string, cfg *ConsumerConfig) (ConsumerSt
 	// Check for encryption.
 	if o.prf != nil {
 		if ekey, err := os.ReadFile(filepath.Join(odir, JetStreamMetaFileKey)); err == nil {
+			if len(ekey) < minBlkKeySize {
+				return nil, errBadKeySize
+			}
 			// Recover key encryption key.
 			rb, err := fs.prf([]byte(fs.cfg.Name + tsep + o.name))
 			if err != nil {
 				return nil, err
 			}
-			kek, err := chacha20poly1305.NewX(rb)
+
+			sc := fs.fcfg.Cipher
+			kek, err := genEncryptionKey(sc, rb)
 			if err != nil {
 				return nil, err
 			}
 			ns := kek.NonceSize()
-			seed, err := kek.Open(nil, ekey[:ns], ekey[ns:], nil)
+			nonce := ekey[:ns]
+			seed, err := kek.Open(nil, nonce, ekey[ns:], nil)
 			if err != nil {
-				return nil, err
+				// We may be here on a cipher conversion, so attempt to convert.
+				if err = o.convertCipher(); err != nil {
+					return nil, err
+				}
+			} else {
+				o.aek, err = genEncryptionKey(sc, seed)
 			}
-			if o.aek, err = chacha20poly1305.NewX(seed); err != nil {
+			if err != nil {
 				return nil, err
 			}
 		}
@@ -5640,6 +5786,63 @@ func (fs *fileStore) ConsumerStore(name string, cfg *ConsumerConfig) (ConsumerSt
 	return o, nil
 }
 
+func (o *consumerFileStore) convertCipher() error {
+	fs := o.fs
+	odir := filepath.Join(fs.fcfg.StoreDir, consumerDir, o.name)
+
+	ekey, err := os.ReadFile(filepath.Join(odir, JetStreamMetaFileKey))
+	if err != nil {
+		return err
+	}
+	if len(ekey) < minBlkKeySize {
+		return errBadKeySize
+	}
+	// Recover key encryption key.
+	rb, err := fs.prf([]byte(fs.cfg.Name + tsep + o.name))
+	if err != nil {
+		return err
+	}
+
+	// Do these in reverse since converting.
+	sc := fs.fcfg.Cipher
+	osc := AES
+	if sc == AES {
+		osc = ChaCha
+	}
+	kek, err := genEncryptionKey(osc, rb)
+	if err != nil {
+		return err
+	}
+	ns := kek.NonceSize()
+	nonce := ekey[:ns]
+	seed, err := kek.Open(nil, nonce, ekey[ns:], nil)
+	if err != nil {
+		return err
+	}
+	aek, err := genEncryptionKey(osc, seed)
+	if err != nil {
+		return err
+	}
+	// Now read in and decode our state using the old cipher.
+	buf, err := os.ReadFile(o.ifn)
+	if err != nil {
+		return err
+	}
+	buf, err = aek.Open(nil, buf[:ns], buf[ns:], nil)
+	if err != nil {
+		return err
+	}
+
+	// Since we are here we recovered our old state.
+	// Now write our meta, which will generate the new keys with the new cipher.
+	if err := o.writeConsumerMeta(); err != nil {
+		return err
+	}
+
+	// Now write out or state with the new cipher.
+	return o.writeState(buf)
+}
+
 // Kick flusher for this consumer.
 // Lock should be held.
 func (o *consumerFileStore) kickFlusher() {
diff --git a/server/jetstream.go b/server/jetstream.go
index 24580346..0eb03aec 100644
--- a/server/jetstream.go
+++ b/server/jetstream.go
@@ -19,7 +19,6 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math"
 	"os"
@@ -34,7 +33,6 @@ import (
 	"github.com/nats-io/nats-server/v2/server/sysmem"
 	"github.com/nats-io/nkeys"
 	"github.com/nats-io/nuid"
-	"golang.org/x/crypto/chacha20poly1305"
 )
 
 // JetStreamConfig determines this server's configuration.
@@ -202,10 +200,6 @@ func (s *Server) EnableJetStream(config *JetStreamConfig) error {
 		return err
 	}
 
-	if ek := s.getOpts().JetStreamKey; ek != _EMPTY_ {
-		s.Warnf("JetStream Encryption is Beta")
-	}
-
 	return s.enableJetStream(cfg)
 }
 
@@ -231,9 +225,9 @@ func (s *Server) jsKeyGen(info string) keyGen {
 }
 
 // Decode the encrypted metafile.
-func (s *Server) decryptMeta(ekey, buf []byte, acc, context string) ([]byte, error) {
-	if len(ekey) != metaKeySize {
-		return nil, errors.New("bad encryption key")
+func (s *Server) decryptMeta(sc StoreCipher, ekey, buf []byte, acc, context string) ([]byte, error) {
+	if len(ekey) < minMetaKeySize {
+		return nil, errBadKeySize
 	}
 	prf := s.jsKeyGen(acc)
 	if prf == nil {
@@ -243,7 +237,8 @@ func (s *Server) decryptMeta(ekey, buf []byte, acc, context string) ([]byte, err
 	if err != nil {
 		return nil, err
 	}
-	kek, err := chacha20poly1305.NewX(rb)
+
+	kek, err := genEncryptionKey(sc, rb)
 	if err != nil {
 		return nil, err
 	}
@@ -252,11 +247,10 @@ func (s *Server) decryptMeta(ekey, buf []byte, acc, context string) ([]byte, err
 	if err != nil {
 		return nil, err
 	}
-	aek, err := chacha20poly1305.NewX(seed[:])
+	aek, err := genEncryptionKey(sc, seed)
 	if err != nil {
 		return nil, err
 	}
-	ns = kek.NonceSize()
 	plain, err := aek.Open(nil, buf[:ns], buf[ns:], nil)
 	if err != nil {
 		return nil, err
@@ -371,6 +365,10 @@ func (s *Server) enableJetStream(cfg JetStreamConfig) error {
 	if cfg.Domain != _EMPTY_ {
 		s.Noticef("  Domain:          %s", cfg.Domain)
 	}
+	opts := s.getOpts()
+	if ek := opts.JetStreamKey; ek != _EMPTY_ {
+		s.Noticef("  Encryption:      %s", opts.JetStreamCipher)
+	}
 	s.Noticef("-------------------------------------------")
 
 	// Setup our internal subscriptions.
@@ -1141,6 +1139,11 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits) erro
 	}
 	var consumers []*ce
 
+	// Remember if we should be encrypted and what cipher we think we should use.
+	encrypted := s.getOpts().JetStreamKey != _EMPTY_
+	plaintext := true
+	sc := s.getOpts().JetStreamCipher
+
 	// Now recover the streams.
 	fis, _ := os.ReadDir(sdir)
 	for _, fi := range fis {
@@ -1177,18 +1180,40 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits) erro
 			continue
 		}
 
+		// Track if we are converting ciphers.
+		var osc StoreCipher
+		var convertingCiphers bool
+
 		// Check if we are encrypted.
-		if key, err := os.ReadFile(filepath.Join(mdir, JetStreamMetaFileKey)); err == nil {
+		keyFile := filepath.Join(mdir, JetStreamMetaFileKey)
+		if key, err := os.ReadFile(keyFile); err == nil {
 			s.Debugf("  Stream metafile is encrypted, reading encrypted keyfile")
-			if len(key) != metaKeySize {
+			if len(key) < minMetaKeySize {
 				s.Warnf("  Bad stream encryption key length of %d", len(key))
 				continue
 			}
 			// Decode the buffer before proceeding.
-			if buf, err = s.decryptMeta(key, buf, a.Name, fi.Name()); err != nil {
-				s.Warnf("  Error decrypting our stream metafile: %v", err)
-				continue
+			nbuf, err := s.decryptMeta(sc, key, buf, a.Name, fi.Name())
+			if err != nil {
+				// See if we are changing ciphers.
+				switch sc {
+				case ChaCha:
+					nbuf, err = s.decryptMeta(AES, key, buf, a.Name, fi.Name())
+					osc, convertingCiphers = AES, true
+				case AES:
+					nbuf, err = s.decryptMeta(ChaCha, key, buf, a.Name, fi.Name())
+					osc, convertingCiphers = ChaCha, true
+				}
+				if err != nil {
+					s.Warnf("  Error decrypting our stream metafile: %v", err)
+					continue
+				}
 			}
+			buf = nbuf
+			plaintext = false
+
+			// Remove the key file to have system regenerate with the new cipher.
+			os.Remove(keyFile)
 		}
 
 		var cfg FileStreamInfo
@@ -1234,6 +1259,15 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits) erro
 
 		s.Noticef("  Starting restore for stream '%s > %s'", a.Name, cfg.StreamConfig.Name)
 
+		// Log if we are converting from plaintext to encrypted.
+		if encrypted {
+			if plaintext {
+				s.Noticef("  Encrypting stream '%s > %s'", a.Name, cfg.StreamConfig.Name)
+			} else if convertingCiphers {
+				s.Noticef("  Converting from %s to %s for stream '%s > %s'", osc, sc, a.Name, cfg.StreamConfig.Name)
+			}
+		}
+
 		// Add in the stream.
 		mset, err := a.addStream(&cfg.StreamConfig)
 		if err != nil {
@@ -1278,10 +1312,22 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits) erro
 			if key, err := os.ReadFile(filepath.Join(e.odir, ofi.Name(), JetStreamMetaFileKey)); err == nil {
 				s.Debugf("  Consumer metafile is encrypted, reading encrypted keyfile")
 				// Decode the buffer before proceeding.
-				if buf, err = s.decryptMeta(key, buf, a.Name, e.mset.name()+tsep+ofi.Name()); err != nil {
-					s.Warnf("  Error decrypting our consumer metafile: %v", err)
-					continue
+				ctxName := e.mset.name() + tsep + ofi.Name()
+				nbuf, err := s.decryptMeta(sc, key, buf, a.Name, ctxName)
+				if err != nil {
+					// See if we are changing ciphers.
+					switch sc {
+					case ChaCha:
+						nbuf, err = s.decryptMeta(AES, key, buf, a.Name, ctxName)
+					case AES:
+						nbuf, err = s.decryptMeta(ChaCha, key, buf, a.Name, ctxName)
+					}
+					if err != nil {
+						s.Warnf("  Error decrypting our consumer metafile: %v", err)
+						continue
+					}
 				}
+				buf = nbuf
 			}
 
 			var cfg FileConsumerInfo
diff --git a/server/opts.go b/server/opts.go
index fe2ab2b4..ecfa1d5e 100644
--- a/server/opts.go
+++ b/server/opts.go
@@ -252,6 +252,7 @@ type Options struct {
 	JetStreamDomain       string        `json:"-"`
 	JetStreamExtHint      string        `json:"-"`
 	JetStreamKey          string        `json:"-"`
+	JetStreamCipher       StoreCipher   `json:"-"`
 	JetStreamUniqueTag    string
 	JetStreamLimits       JSLimitOpts
 	StoreDir              string            `json:"-"`
@@ -1930,6 +1931,15 @@ func parseJetStream(v interface{}, opts *Options, errors *[]error, warnings *[]e
 				doEnable = mv.(bool)
 			case "key", "ek", "encryption_key":
 				opts.JetStreamKey = mv.(string)
+			case "cipher":
+				switch strings.ToLower(mv.(string)) {
+				case "chacha", "chachapoly":
+					opts.JetStreamCipher = ChaCha
+				case "aes":
+					opts.JetStreamCipher = AES
+				default:
+					return &configErr{tk, fmt.Sprintf("Unknown cipher type: %q", mv)}
+				}
 			case "extension_hint":
 				opts.JetStreamExtHint = mv.(string)
 			case "limits":
diff --git a/server/reload.go b/server/reload.go
index 372414e1..0b23b298 100644
--- a/server/reload.go
+++ b/server/reload.go
@@ -945,7 +945,7 @@ func imposeOrder(value interface{}) error {
 		sort.Strings(value.AllowedOrigins)
 	case string, bool, uint8, int, int32, int64, time.Duration, float64, nil, LeafNodeOpts, ClusterOpts, *tls.Config, PinnedCertSet,
 		*URLAccResolver, *MemAccResolver, *DirAccResolver, *CacheDirAccResolver, Authentication, MQTTOpts, jwt.TagList,
-		*OCSPConfig, map[string]string, JSLimitOpts:
+		*OCSPConfig, map[string]string, JSLimitOpts, StoreCipher:
 		// explicitly skipped types
 	default:
 		// this will fail during unit tests
@@ -976,7 +976,7 @@ func (s *Server) diffOptions(newOpts *Options) ([]option, error) {
 		field := oldConfig.Type().Field(i)
 		// field.PkgPath is empty for exported fields, and is not for unexported ones.
 		// We skip the unexported fields.
-		if field.PkgPath != "" {
+		if field.PkgPath != _EMPTY_ {
 			continue
 		}
 		var (
diff --git a/server/stream.go b/server/stream.go
index e85c68bd..a7bb231a 100644
--- a/server/stream.go
+++ b/server/stream.go
@@ -3168,7 +3168,12 @@ func (mset *stream) setupStore(fsCfg *FileStoreConfig) error {
 		mset.store = ms
 	case FileStorage:
 		s := mset.srv
-		fs, err := newFileStoreWithCreated(*fsCfg, mset.cfg, mset.created, s.jsKeyGen(mset.acc.Name))
+		prf := s.jsKeyGen(mset.acc.Name)
+		if prf != nil {
+			// We are encrypted here, fill in correct cipher selection.
+			fsCfg.Cipher = s.getOpts().JetStreamCipher
+		}
+		fs, err := newFileStoreWithCreated(*fsCfg, mset.cfg, mset.created, prf)
 		if err != nil {
 			mset.mu.Unlock()
 			return err


Test patch:
diff --git a/server/filestore_test.go b/server/filestore_test.go
index b3d0e7db..33db1505 100644
--- a/server/filestore_test.go
+++ b/server/filestore_test.go
@@ -4099,7 +4099,7 @@ func TestFileStoreEncryptedKeepIndexNeedBekResetBug(t *testing.T) {
 		fs.StoreMsg(subj, nil, msg)
 	}
 
-	// What to go to 0.
+	// Want to go to 0.
 	// This will leave the marker.
 	checkFor(t, time.Second, ttl, func() error {
 		if state := fs.State(); state.Msgs != 0 {
@@ -4301,5 +4301,68 @@ func TestFileStoreSubjectStateCacheExpiration(t *testing.T) {
 	if ss := fss["kv.foo.1"]; ss != expected {
 		t.Fatalf("Bad subject state, expected %+v but got %+v", expected, ss)
 	}
+}
+
+func TestFileStoreEncryptedAES(t *testing.T) {
+	storeDir := createDir(t, JetStreamStoreDir)
+	defer os.RemoveAll(storeDir)
 
+	prf := func(context []byte) ([]byte, error) {
+		h := hmac.New(sha256.New, []byte("dlc22"))
+		if _, err := h.Write(context); err != nil {
+			return nil, err
+		}
+		return h.Sum(nil), nil
+	}
+
+	fs, err := newFileStoreWithCreated(
+		FileStoreConfig{StoreDir: storeDir, Cipher: AES},
+		StreamConfig{Name: "zzz", Storage: FileStorage},
+		time.Now(),
+		prf,
+	)
+	require_NoError(t, err)
+	defer fs.Stop()
+
+	subj, msg := "foo", []byte("aes ftw")
+	for i := 0; i < 50; i++ {
+		fs.StoreMsg(subj, nil, msg)
+	}
+
+	o, err := fs.ConsumerStore("o22", &ConsumerConfig{})
+	require_NoError(t, err)
+
+	state := &ConsumerState{}
+	state.Delivered.Consumer = 22
+	state.Delivered.Stream = 22
+	state.AckFloor.Consumer = 11
+	state.AckFloor.Stream = 11
+	err = o.Update(state)
+	require_NoError(t, err)
+
+	fs.Stop()
+
+	fs, err = newFileStoreWithCreated(
+		FileStoreConfig{StoreDir: storeDir, Cipher: AES},
+		StreamConfig{Name: "zzz", Storage: FileStorage},
+		time.Now(),
+		prf,
+	)
+	require_NoError(t, err)
+	defer fs.Stop()
+
+	// Now make sure we can read.
+	var smv StoreMsg
+	sm, err := fs.LoadMsg(10, &smv)
+	require_NoError(t, err)
+	require_True(t, string(sm.msg) == "aes ftw")
+
+	o, err = fs.ConsumerStore("o22", &ConsumerConfig{})
+	require_NoError(t, err)
+	rstate, err := o.State()
+	require_NoError(t, err)
+
+	if rstate.Delivered != state.Delivered || rstate.AckFloor != state.AckFloor {
+		t.Fatalf("Bad recovered consumer state, expected %+v got %+v", state, rstate)
+	}
 }
diff --git a/server/jetstream_chaos_consumer_test.go b/server/jetstream_chaos_consumer_test.go
deleted file mode 100644
index bd21104b..00000000
--- a/server/jetstream_chaos_consumer_test.go
+++ /dev/null
@@ -1,615 +0,0 @@
-// Copyright 2022 The NATS Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build js_chaos_tests
-// +build js_chaos_tests
-
-package server
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/nats-io/nats.go"
-)
-
-const (
-	chaosConsumerTestsClusterName = "CONSUMERS_CHAOS_TEST"
-	chaosConsumerTestsStreamName  = "CONSUMER_CHAOS_TEST_STREAM"
-	chaosConsumerTestsSubject     = "foo"
-	chaosConsumerTestsDebug       = false
-)
-
-// Creates stream and fills it with the given number of messages.
-// Each message is the string representation of the stream sequence number,
-// e.g. the first message (seqno: 1) contains data "1".
-// This allows consumers to verify the content of each message without tracking additional state
-func createStreamForConsumerChaosTest(t *testing.T, c *cluster, replicas, numMessages int) {
-	t.Helper()
-
-	const publishBatchSize = 1_000
-
-	pubNc, pubJs := jsClientConnectCluster(t, c)
-	defer pubNc.Close()
-
-	_, err := pubJs.AddStream(&nats.StreamConfig{
-		Name:     chaosConsumerTestsStreamName,
-		Subjects: []string{chaosConsumerTestsSubject},
-		Replicas: replicas,
-	})
-	if err != nil {
-		t.Fatalf("Error creating stream: %v", err)
-	}
-
-	ackFutures := make([]nats.PubAckFuture, 0, publishBatchSize)
-
-	for i := 1; i <= numMessages; i++ {
-		message := []byte(fmt.Sprintf("%d", i))
-		pubAckFuture, err := pubJs.PublishAsync(chaosConsumerTestsSubject, message, nats.ExpectLastSequence(uint64(i-1)))
-		if err != nil {
-			t.Fatalf("Publish error: %s", err)
-		}
-		ackFutures = append(ackFutures, pubAckFuture)
-
-		if (i > 0 && i%publishBatchSize == 0) || i == numMessages {
-			select {
-			case <-pubJs.PublishAsyncComplete():
-				for _, pubAckFuture := range ackFutures {
-					select {
-					case <-pubAckFuture.Ok():
-						// Noop
-					case pubAckErr := <-pubAckFuture.Err():
-						t.Fatalf("Error publishing: %s", pubAckErr)
-					case <-time.After(30 * time.Second):
-						t.Fatalf("Timeout verifying pubAck for message: %s", pubAckFuture.Msg().Data)
-					}
-				}
-				ackFutures = make([]nats.PubAckFuture, 0, publishBatchSize)
-				t.Logf("Published %d/%d messages", i, numMessages)
-
-			case <-time.After(30 * time.Second):
-				t.Fatalf("Publish timed out")
-			}
-		}
-	}
-}
-
-// Verify ordered delivery despite cluster-wide outages
-func TestJetStreamChaosConsumerOrdered(t *testing.T) {
-
-	const numMessages = 30_000
-	const maxRetries = 100
-	const retryDelay = 500 * time.Millisecond
-	const fetchTimeout = 250 * time.Millisecond
-	const clusterSize = 3
-	const replicas = 3
-
-	c := createJetStreamClusterExplicit(t, chaosConsumerTestsClusterName, clusterSize)
-	defer c.shutdown()
-
-	createStreamForConsumerChaosTest(t, c, replicas, numMessages)
-
-	chaos := createClusterChaosMonkeyController(
-		t,
-		c,
-		&clusterBouncerChaosMonkey{
-			minDowntime:    0 * time.Second,
-			maxDowntime:    2 * time.Second,
-			minDownServers: clusterSize, // Whole cluster outage
-			maxDownServers: clusterSize,
-			pause:          1 * time.Second,
-		},
-	)
-
-	subNc, subJs := jsClientConnectCluster(t, c)
-	defer subNc.Close()
-
-	sub, err := subJs.SubscribeSync(
-		chaosConsumerTestsSubject,
-		nats.OrderedConsumer(),
-	)
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	defer sub.Unsubscribe()
-
-	if chaosConsumerTestsDebug {
-		t.Logf("Initial subscription: %s", toIndentedJsonString(sub))
-	}
-
-	chaos.start()
-	defer chaos.stop()
-
-	for i := 1; i <= numMessages; i++ {
-		var msg *nats.Msg
-		var nextMsgErr error
-		var expectedMsgData = []byte(fmt.Sprintf("%d", i))
-
-	nextMsgRetryLoop:
-		for r := 0; r <= maxRetries; r++ {
-			msg, nextMsgErr = sub.NextMsg(fetchTimeout)
-			if nextMsgErr == nil {
-				break nextMsgRetryLoop
-			} else if r == maxRetries {
-				t.Fatalf("Exceeded max retries for NextMsg")
-			} else if nextMsgErr == nats.ErrBadSubscription {
-				t.Fatalf("Subscription is invalid: %s", toIndentedJsonString(sub))
-			} else {
-				time.Sleep(retryDelay)
-			}
-		}
-
-		metadata, err := msg.Metadata()
-		if err != nil {
-			t.Fatalf("Failed to get message metadata: %v", err)
-		}
-
-		if metadata.Sequence.Stream != uint64(i) {
-			t.Fatalf("Expecting stream sequence %d, got %d instead", i, metadata.Sequence.Stream)
-		}
-
-		if !bytes.Equal(msg.Data, expectedMsgData) {
-			t.Fatalf("Expecting message %s, got %s instead", expectedMsgData, msg.Data)
-		}
-
-		// Simulate application processing (and gives the monkey some time to brew chaos)
-		time.Sleep(10 * time.Millisecond)
-
-		if i%1000 == 0 {
-			t.Logf("Consumed %d/%d", i, numMessages)
-		}
-	}
-}
-
-// Verify ordered delivery despite cluster-wide outages
-func TestJetStreamChaosConsumerAsync(t *testing.T) {
-
-	const numMessages = 30_000
-	const timeout = 30 * time.Second // No (new) messages for 30s => terminate
-	const maxRetries = 25
-	const retryDelay = 500 * time.Millisecond
-	const clusterSize = 3
-	const replicas = 3
-
-	c := createJetStreamClusterExplicit(t, chaosConsumerTestsClusterName, clusterSize)
-	defer c.shutdown()
-
-	createStreamForConsumerChaosTest(t, c, replicas, numMessages)
-
-	chaos := createClusterChaosMonkeyController(
-		t,
-		c,
-		&clusterBouncerChaosMonkey{
-			minDowntime:    0 * time.Second,
-			maxDowntime:    2 * time.Second,
-			minDownServers: clusterSize,
-			maxDownServers: clusterSize,
-			pause:          2 * time.Second,
-		},
-	)
-
-	subNc, subJs := jsClientConnectCluster(t, c)
-	defer subNc.Close()
-
-	timeoutTimer := time.NewTimer(timeout)
-	deliveryCount := uint64(0)
-	received := NewBitset(numMessages)
-
-	handleMsg := func(msg *nats.Msg) {
-		deliveryCount += 1
-
-		metadata, err := msg.Metadata()
-		if err != nil {
-			t.Fatalf("Failed to get message metadata: %v", err)
-		}
-		seq := metadata.Sequence.Stream
-
-		var expectedMsgData = []byte(fmt.Sprintf("%d", seq))
-		if !bytes.Equal(msg.Data, expectedMsgData) {
-			t.Fatalf("Expecting message content '%s', got '%s' instead", expectedMsgData, msg.Data)
-		}
-
-		isDupe := received.get(seq - 1)
-
-		if isDupe {
-			if chaosConsumerTestsDebug {
-				t.Logf("Duplicate message delivery, seq: %d", seq)
-			}
-			return
-		}
-
-		// Mark this sequence as received
-		received.set(seq-1, true)
-		if received.count() < numMessages {
-			// Reset timeout
-			timeoutTimer.Reset(timeout)
-		} else {
-			// All received, speed up the shutdown
-			timeoutTimer.Reset(1 * time.Second)
-		}
-
-		if received.count()%1000 == 0 {
-			t.Logf("Consumed %d/%d", received.count(), numMessages)
-		}
-
-		// Simulate application processing (and gives the monkey some time to brew chaos)
-		time.Sleep(10 * time.Millisecond)
-
-	ackRetryLoop:
-		for i := 0; i <= maxRetries; i++ {
-			ackErr := msg.Ack()
-			if ackErr == nil {
-				break ackRetryLoop
-			} else if i == maxRetries {
-				t.Fatalf("Failed to ACK message %d (retried %d times)", seq, maxRetries)
-			} else {
-				time.Sleep(retryDelay)
-			}
-		}
-	}
-
-	subOpts := []nats.SubOpt{}
-	sub, err := subJs.Subscribe(chaosConsumerTestsSubject, handleMsg, subOpts...)
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	defer sub.Unsubscribe()
-
-	chaos.start()
-	defer chaos.stop()
-
-	// Wait for long enough silence.
-	// Either a stall, or all messages received
-	<-timeoutTimer.C
-
-	// Shut down consumer
-	sub.Unsubscribe()
-
-	uniqueDeliveredCount := received.count()
-
-	t.Logf(
-		"Delivered %d/%d messages %d duplicate deliveries",
-		uniqueDeliveredCount,
-		numMessages,
-		deliveryCount-uniqueDeliveredCount,
-	)
-
-	if uniqueDeliveredCount != numMessages {
-		t.Fatalf("No new message delivered in the last %s, %d/%d messages never delivered", timeout, numMessages-uniqueDeliveredCount, numMessages)
-	}
-}
-
-// Verify durable consumer retains state despite cluster-wide outages
-// The consumer connection is also periodically closed, and the consumer 'resumes' on a different one
-func TestJetStreamChaosConsumerDurable(t *testing.T) {
-
-	const numMessages = 30_000
-	const timeout = 30 * time.Second // No (new) messages for 60s => terminate
-	const clusterSize = 3
-	const replicas = 3
-	const maxRetries = 25
-	const retryDelay = 500 * time.Millisecond
-	const durableConsumerName = "durable"
-
-	c := createJetStreamClusterExplicit(t, chaosConsumerTestsClusterName, clusterSize)
-	defer c.shutdown()
-
-	createStreamForConsumerChaosTest(t, c, replicas, numMessages)
-
-	chaos := createClusterChaosMonkeyController(
-		t,
-		c,
-		&clusterBouncerChaosMonkey{
-			minDowntime:    0 * time.Second,
-			maxDowntime:    2 * time.Second,
-			minDownServers: 1,
-			maxDownServers: clusterSize,
-			pause:          3 * time.Second,
-		},
-	)
-
-	var nc *nats.Conn
-	var sub *nats.Subscription
-	var subLock sync.Mutex
-
-	var handleMsgFun func(msg *nats.Msg)
-	var natsURL string
-
-	{
-		var sb strings.Builder
-		for _, s := range c.servers {
-			sb.WriteString(s.ClientURL())
-			sb.WriteString(",")
-		}
-		natsURL = sb.String()
-	}
-
-	resetDurableConsumer := func() {
-		subLock.Lock()
-		defer subLock.Unlock()
-
-		if nc != nil {
-			nc.Close()
-		}
-
-		var newNc *nats.Conn
-	connectRetryLoop:
-		for r := 0; r <= maxRetries; r++ {
-			var connErr error
-			newNc, connErr = nats.Connect(natsURL)
-			if connErr == nil {
-				break connectRetryLoop
-			} else if r == maxRetries {
-				t.Fatalf("Failed to connect, exceeded max retries, last error: %s", connErr)
-			} else {
-				time.Sleep(retryDelay)
-			}
-		}
-
-		var newJs nats.JetStreamContext
-	jsRetryLoop:
-		for r := 0; r <= maxRetries; r++ {
-			var jsErr error
-			newJs, jsErr = newNc.JetStream(nats.MaxWait(10 * time.Second))
-			if jsErr == nil {
-				break jsRetryLoop
-			} else if r == maxRetries {
-				t.Fatalf("Failed to get JS, exceeded max retries, last error: %s", jsErr)
-			} else {
-				time.Sleep(retryDelay)
-			}
-		}
-
-		subOpts := []nats.SubOpt{
-			nats.Durable(durableConsumerName),
-		}
-
-		var newSub *nats.Subscription
-	subscribeRetryLoop:
-		for i := 0; i <= maxRetries; i++ {
-			var subErr error
-			newSub, subErr = newJs.Subscribe(chaosConsumerTestsSubject, handleMsgFun, subOpts...)
-			if subErr == nil {
-				ci, err := newJs.ConsumerInfo(chaosConsumerTestsStreamName, durableConsumerName)
-				if err == nil {
-					if chaosConsumerTestsDebug {
-						t.Logf("Consumer info:\n %s", toIndentedJsonString(ci))
-					}
-				} else {
-					t.Logf("Failed to retrieve consumer info: %s", err)
-				}
-
-				break subscribeRetryLoop
-			} else if i == maxRetries {
-				t.Fatalf("Exceeded max retries creating subscription: %v", subErr)
-			} else {
-				time.Sleep(retryDelay)
-			}
-		}
-
-		nc, sub = newNc, newSub
-	}
-
-	timeoutTimer := time.NewTimer(timeout)
-	deliveryCount := uint64(0)
-	received := NewBitset(numMessages)
-
-	handleMsgFun = func(msg *nats.Msg) {
-
-		subLock.Lock()
-		if msg.Sub != sub {
-			// Message from a previous instance of durable consumer, drop
-			defer subLock.Unlock()
-			return
-		}
-		subLock.Unlock()
-
-		deliveryCount += 1
-
-		metadata, err := msg.Metadata()
-		if err != nil {
-			t.Fatalf("Failed to get message metadata: %v", err)
-		}
-		seq := metadata.Sequence.Stream
-
-		var expectedMsgData = []byte(fmt.Sprintf("%d", seq))
-		if !bytes.Equal(msg.Data, expectedMsgData) {
-			t.Fatalf("Expecting message content '%s', got '%s' instead", expectedMsgData, msg.Data)
-		}
-
-		isDupe := received.get(seq - 1)
-
-		if isDupe {
-			if chaosConsumerTestsDebug {
-				t.Logf("Duplicate message delivery, seq: %d", seq)
-			}
-			return
-		}
-
-		// Mark this sequence as received
-		received.set(seq-1, true)
-		if received.count() < numMessages {
-			// Reset timeout
-			timeoutTimer.Reset(timeout)
-		} else {
-			// All received, speed up the shutdown
-			timeoutTimer.Reset(1 * time.Second)
-		}
-
-		// Simulate application processing (and gives the monkey some time to brew chaos)
-		time.Sleep(10 * time.Millisecond)
-
-	ackRetryLoop:
-		for i := 0; i <= maxRetries; i++ {
-			ackErr := msg.Ack()
-			if ackErr == nil {
-				break ackRetryLoop
-			} else if i == maxRetries {
-				t.Fatalf("Failed to ACK message %d (retried %d times)", seq, maxRetries)
-			} else {
-				time.Sleep(retryDelay)
-			}
-		}
-
-		if received.count()%1000 == 0 {
-			t.Logf("Consumed %d/%d, duplicate deliveries: %d", received.count(), numMessages, deliveryCount-received.count())
-			// Close connection and resume consuming on a different one
-			resetDurableConsumer()
-		}
-	}
-
-	resetDurableConsumer()
-
-	chaos.start()
-	defer chaos.stop()
-
-	// Wait for long enough silence.
-	// Either a stall, or all messages received
-	<-timeoutTimer.C
-
-	// Shut down consumer
-	if sub != nil {
-		sub.Unsubscribe()
-	}
-
-	uniqueDeliveredCount := received.count()
-
-	t.Logf(
-		"Delivered %d/%d messages %d duplicate deliveries",
-		uniqueDeliveredCount,
-		numMessages,
-		deliveryCount-uniqueDeliveredCount,
-	)
-
-	if uniqueDeliveredCount != numMessages {
-		t.Fatalf("No new message delivered in the last %s, %d/%d messages never delivered", timeout, numMessages-uniqueDeliveredCount, numMessages)
-	}
-}
-
-func TestJetStreamChaosConsumerPull(t *testing.T) {
-
-	const numMessages = 10_000
-	const maxRetries = 100
-	const retryDelay = 500 * time.Millisecond
-	const fetchTimeout = 250 * time.Millisecond
-	const fetchBatchSize = 100
-	const clusterSize = 3
-	const replicas = 3
-	const durableConsumerName = "durable"
-
-	c := createJetStreamClusterExplicit(t, chaosConsumerTestsClusterName, clusterSize)
-	defer c.shutdown()
-
-	createStreamForConsumerChaosTest(t, c, replicas, numMessages)
-
-	chaos := createClusterChaosMonkeyController(
-		t,
-		c,
-		&clusterBouncerChaosMonkey{
-			minDowntime:    0 * time.Second,
-			maxDowntime:    2 * time.Second,
-			minDownServers: clusterSize, // Whole cluster outage
-			maxDownServers: clusterSize,
-			pause:          1 * time.Second,
-		},
-	)
-
-	subNc, subJs := jsClientConnectCluster(t, c)
-	defer subNc.Close()
-
-	sub, err := subJs.PullSubscribe(
-		chaosConsumerTestsSubject,
-		durableConsumerName,
-	)
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	defer sub.Unsubscribe()
-
-	if chaosConsumerTestsDebug {
-		t.Logf("Initial subscription: %s", toIndentedJsonString(sub))
-	}
-
-	chaos.start()
-	defer chaos.stop()
-
-	fetchMaxWait := nats.MaxWait(fetchTimeout)
-	received := NewBitset(numMessages)
-	deliveredCount := uint64(0)
-
-	for received.count() < numMessages {
-
-		var msgs []*nats.Msg
-		var fetchErr error
-
-	fetchRetryLoop:
-		for r := 0; r <= maxRetries; r++ {
-			msgs, fetchErr = sub.Fetch(fetchBatchSize, fetchMaxWait)
-			if fetchErr == nil {
-				break fetchRetryLoop
-			} else if r == maxRetries {
-				t.Fatalf("Exceeded max retries for Fetch, last error: %s", fetchErr)
-			} else if fetchErr == nats.ErrBadSubscription {
-				t.Fatalf("Subscription is invalid: %s", toIndentedJsonString(sub))
-			} else {
-				// t.Logf("Fetch error: %v", fetchErr)
-				time.Sleep(retryDelay)
-			}
-		}
-
-		for _, msg := range msgs {
-
-			deliveredCount += 1
-
-			metadata, err := msg.Metadata()
-			if err != nil {
-				t.Fatalf("Failed to get message metadata: %v", err)
-			}
-
-			streamSeq := metadata.Sequence.Stream
-
-			expectedMsgData := []byte(fmt.Sprintf("%d", streamSeq))
-
-			if !bytes.Equal(msg.Data, expectedMsgData) {
-				t.Fatalf("Expecting message %s, got %s instead", expectedMsgData, msg.Data)
-			}
-
-			isDupe := received.get(streamSeq - 1)
-
-			received.set(streamSeq-1, true)
-
-			// Simulate application processing (and gives the monkey some time to brew chaos)
-			time.Sleep(10 * time.Millisecond)
-
-		ackRetryLoop:
-			for r := 0; r <= maxRetries; r++ {
-				ackErr := msg.Ack()
-				if ackErr == nil {
-					break ackRetryLoop
-				} else if r == maxRetries {
-					t.Fatalf("Failed to ACK message %d, last error: %s", streamSeq, ackErr)
-				} else {
-					time.Sleep(retryDelay)
-				}
-			}
-
-			if !isDupe && received.count()%1000 == 0 {
-				t.Logf("Consumed %d/%d (duplicates: %d)", received.count(), numMessages, deliveredCount-received.count())
-			}
-		}
-	}
-}
diff --git a/server/jetstream_chaos_helpers_test.go b/server/jetstream_chaos_helpers_test.go
index 4c89adea..5b3f9001 100644
--- a/server/jetstream_chaos_helpers_test.go
+++ b/server/jetstream_chaos_helpers_test.go
@@ -17,10 +17,8 @@
 package server
 
 import (
-	"encoding/json"
 	"fmt"
 	"math/rand"
-	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -177,90 +175,3 @@ func (m *clusterBouncerChaosMonkey) run(t *testing.T, c *cluster, stopCh <-chan
 		c.waitOnLeader()
 	}
 }
-
-// Bitset, aka bitvector, allows tracking of large number of bits efficiently
-type bitset struct {
-	// Bit map storage
-	bitmap []uint8
-	// Number of bits currently set to 1
-	currentCount uint64
-	// Number of bits stored
-	size uint64
-}
-
-func NewBitset(size uint64) *bitset {
-	byteSize := (size + 7) / 8 //Round up to the nearest byte
-
-	return &bitset{
-		bitmap:       make([]uint8, int(byteSize)),
-		size:         size,
-		currentCount: 0,
-	}
-}
-
-func (b *bitset) get(index uint64) bool {
-	if index >= b.size {
-		panic(fmt.Sprintf("Index %d out of bounds, size %d", index, b.size))
-	}
-	byteIndex := index / 8
-	bitIndex := uint(index % 8)
-	bit := (b.bitmap[byteIndex] & (uint8(1) << bitIndex))
-	return bit != 0
-}
-
-func (b *bitset) set(index uint64, value bool) {
-	if index >= b.size {
-		panic(fmt.Sprintf("Index %d out of bounds, size %d", index, b.size))
-	}
-	byteIndex := index / 8
-	bitIndex := uint(index % 8)
-	byteMask := uint8(1) << bitIndex
-	isSet := (b.bitmap[byteIndex] & (uint8(1) << bitIndex)) != 0
-	if value {
-		b.bitmap[byteIndex] |= byteMask
-		if !isSet {
-			b.currentCount += 1
-		}
-	} else {
-		b.bitmap[byteIndex] &= ^byteMask
-		if isSet {
-			b.currentCount -= 1
-		}
-	}
-}
-
-func (b *bitset) count() uint64 {
-	return b.currentCount
-}
-
-func (b *bitset) String() string {
-	const block = 8 // 8 bytes, 64 bits per line
-	sb := strings.Builder{}
-
-	sb.WriteString(fmt.Sprintf("Bits set: %d/%d\n", b.currentCount, b.size))
-	for i := 0; i < len(b.bitmap); i++ {
-		if i%block == 0 {
-			if i > 0 {
-				sb.WriteString("\n")
-			}
-			sb.WriteString(fmt.Sprintf("[%4d] ", i*8))
-		}
-		for j := uint8(0); j < 8; j++ {
-			if b.bitmap[i]&(1<<j) > 0 {
-				sb.WriteString("1")
-			} else {
-				sb.WriteString("0")
-			}
-		}
-	}
-	sb.WriteString("\n")
-	return sb.String()
-}
-
-func toIndentedJsonString(v interface{}) string {
-	jsonBytes, err := json.MarshalIndent(v, "", "  ")
-	if err != nil {
-		return fmt.Sprintf("Marshal error: %s", err)
-	}
-	return string(jsonBytes)
-}
diff --git a/server/jetstream_cluster_test.go b/server/jetstream_cluster_test.go
index 0d86ebe9..1368dc19 100644
--- a/server/jetstream_cluster_test.go
+++ b/server/jetstream_cluster_test.go
@@ -7229,7 +7229,7 @@ func TestJetStreamClusterLeafDifferentAccounts(t *testing.T) {
 	nc, js := jsClientConnect(t, ln.randomServer())
 	defer nc.Close()
 
-	// Make sure we can properly identify the right account when the leader received the request.
+	// Make sure we can properly indentify the right account when the leader received the request.
 	// We need to map the client info header to the new account once received by the hub.
 	if _, err := js.AccountInfo(); err != nil {
 		t.Fatalf("Unexpected error: %v", err)
diff --git a/server/jetstream_helpers_test.go b/server/jetstream_helpers_test.go
index 954cdbb5..96f8691f 100644
--- a/server/jetstream_helpers_test.go
+++ b/server/jetstream_helpers_test.go
@@ -1068,33 +1068,6 @@ func jsClientConnectEx(t *testing.T, s *Server, domain string, opts ...nats.Opti
 	return nc, js
 }
 
-func jsClientConnectCluster(t *testing.T, c *cluster, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) {
-	t.Helper()
-
-	var sb strings.Builder
-
-	for _, s := range c.servers {
-		sb.WriteString(s.ClientURL())
-		sb.WriteString(",")
-	}
-
-	return jsClientConnectURL(t, sb.String(), opts...)
-}
-
-func jsClientConnectURL(t *testing.T, url string, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) {
-	t.Helper()
-
-	nc, err := nats.Connect(url, opts...)
-	if err != nil {
-		t.Fatalf("Failed to create client: %v", err)
-	}
-	js, err := nc.JetStream(nats.MaxWait(10 * time.Second))
-	if err != nil {
-		t.Fatalf("Unexpected error getting JetStream context: %v", err)
-	}
-	return nc, js
-}
-
 func checkSubsPending(t *testing.T, sub *nats.Subscription, numExpected int) {
 	t.Helper()
 	checkFor(t, 10*time.Second, 20*time.Millisecond, func() error {
diff --git a/server/jetstream_test.go b/server/jetstream_test.go
index 0e6ce28f..1a26dd7c 100644
--- a/server/jetstream_test.go
+++ b/server/jetstream_test.go
@@ -12338,203 +12338,218 @@ func TestJetStreamTemplatedErrorsBug(t *testing.T) {
 }
 
 func TestJetStreamServerEncryption(t *testing.T) {
-	tmpl := `
-		listen: 127.0.0.1:-1
-		jetstream: {key: $JS_KEY, store_dir: '%s'}
-	`
-	storeDir := createDir(t, JetStreamStoreDir)
-	defer removeDir(t, storeDir)
+	cases := []struct {
+		name   string
+		cstr   string
+		cipher StoreCipher
+	}{
+		{"Default", _EMPTY_, ChaCha},
+		{"ChaCha", ", cipher: chacha", ChaCha},
+		{"AES", ", cipher: aes", AES},
+	}
 
-	conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir)))
-	defer removeFile(t, conf)
+	for _, c := range cases {
 
-	os.Setenv("JS_KEY", "s3cr3t!!")
-	defer os.Unsetenv("JS_KEY")
+		t.Run(c.name, func(t *testing.T) {
+			tmpl := `
+				listen: 127.0.0.1:-1
+				jetstream: {key: $JS_KEY, store_dir: '%s' %s}
+			`
+			storeDir := createDir(t, JetStreamStoreDir)
+			defer removeDir(t, storeDir)
 
-	s, _ := RunServerWithConfig(conf)
-	defer s.Shutdown()
+			conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, c.cstr)))
+			defer removeFile(t, conf)
 
-	config := s.JetStreamConfig()
-	if config == nil {
-		t.Fatalf("Expected config but got none")
-	}
-	defer removeDir(t, config.StoreDir)
+			os.Setenv("JS_KEY", "s3cr3t!!")
+			defer os.Unsetenv("JS_KEY")
 
-	// Client based API
-	nc, js := jsClientConnect(t, s)
-	defer nc.Close()
+			s, _ := RunServerWithConfig(conf)
+			defer s.Shutdown()
 
-	cfg := &nats.StreamConfig{
-		Name:     "TEST",
-		Subjects: []string{"foo", "bar", "baz"},
-	}
-	if _, err := js.AddStream(cfg); err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
+			config := s.JetStreamConfig()
+			if config == nil {
+				t.Fatalf("Expected config but got none")
+			}
+			defer removeDir(t, config.StoreDir)
 
-	msg := []byte("ENCRYPTED PAYLOAD!!")
-	sendMsg := func(subj string) {
-		t.Helper()
-		if _, err := js.Publish(subj, msg); err != nil {
-			t.Fatalf("Unexpected publish error: %v", err)
-		}
-	}
-	// Send 10 msgs
-	for i := 0; i < 10; i++ {
-		sendMsg("foo")
-	}
+			// Client based API
+			nc, js := jsClientConnect(t, s)
+			defer nc.Close()
 
-	// Now create a consumer.
-	sub, err := js.PullSubscribe("foo", "dlc")
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	for i, m := range fetchMsgs(t, sub, 10, 5*time.Second) {
-		if i < 5 {
-			m.AckSync()
-		}
-	}
+			cfg := &nats.StreamConfig{
+				Name:     "TEST",
+				Subjects: []string{"foo", "bar", "baz"},
+			}
+			if _, err := js.AddStream(cfg); err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
 
-	// Grab our state to compare after restart.
-	si, _ := js.StreamInfo("TEST")
-	ci, _ := js.ConsumerInfo("TEST", "dlc")
+			msg := []byte("ENCRYPTED PAYLOAD!!")
+			sendMsg := func(subj string) {
+				t.Helper()
+				if _, err := js.Publish(subj, msg); err != nil {
+					t.Fatalf("Unexpected publish error: %v", err)
+				}
+			}
+			// Send 10 msgs
+			for i := 0; i < 10; i++ {
+				sendMsg("foo")
+			}
 
-	// Quick check to make sure everything not just plaintext still.
-	sdir := filepath.Join(config.StoreDir, "$G", "streams", "TEST")
-	// Make sure we can not find any plaintext strings in the target file.
-	checkFor := func(fn string, strs ...string) {
-		t.Helper()
-		data, err := os.ReadFile(fn)
-		if err != nil {
-			t.Fatalf("Unexpected error: %v", err)
-		}
-		for _, str := range strs {
-			if bytes.Contains(data, []byte(str)) {
-				t.Fatalf("Found %q in body of file contents", str)
+			// Now create a consumer.
+			sub, err := js.PullSubscribe("foo", "dlc")
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+			for i, m := range fetchMsgs(t, sub, 10, 5*time.Second) {
+				if i < 5 {
+					m.AckSync()
+				}
 			}
-		}
-	}
-	checkKeyFile := func(fn string) {
-		t.Helper()
-		if _, err := os.Stat(fn); err != nil {
-			t.Fatalf("Expected a key file at %q", fn)
-		}
-	}
 
-	// Check stream meta.
-	checkEncrypted := func() {
-		checkKeyFile(filepath.Join(sdir, JetStreamMetaFileKey))
-		checkFor(filepath.Join(sdir, JetStreamMetaFile), "TEST", "foo", "bar", "baz", "max_msgs", "max_bytes")
-		// Check a message block.
-		checkKeyFile(filepath.Join(sdir, "msgs", "1.key"))
-		checkFor(filepath.Join(sdir, "msgs", "1.blk"), "ENCRYPTED PAYLOAD!!", "foo", "bar", "baz")
+			// Grab our state to compare after restart.
+			si, _ := js.StreamInfo("TEST")
+			ci, _ := js.ConsumerInfo("TEST", "dlc")
 
-		// Check consumer meta and state.
-		checkKeyFile(filepath.Join(sdir, "obs", "dlc", JetStreamMetaFileKey))
-		checkFor(filepath.Join(sdir, "obs", "dlc", JetStreamMetaFile), "TEST", "dlc", "foo", "bar", "baz", "max_msgs", "ack_policy")
-		// Load and see if we can parse the consumer state.
-		state, err := os.ReadFile(filepath.Join(sdir, "obs", "dlc", "o.dat"))
-		if err != nil {
-			t.Fatalf("Unexpected error: %v", err)
-		}
-		if _, err := decodeConsumerState(state); err == nil {
-			t.Fatalf("Expected decoding consumer state to fail")
-		}
-	}
+			// Quick check to make sure everything not just plaintext still.
+			sdir := filepath.Join(config.StoreDir, "$G", "streams", "TEST")
+			// Make sure we can not find any plaintext strings in the target file.
+			checkFor := func(fn string, strs ...string) {
+				t.Helper()
+				data, err := os.ReadFile(fn)
+				if err != nil {
+					t.Fatalf("Unexpected error: %v", err)
+				}
+				for _, str := range strs {
+					if bytes.Contains(data, []byte(str)) {
+						t.Fatalf("Found %q in body of file contents", str)
+					}
+				}
+			}
+			checkKeyFile := func(fn string) {
+				t.Helper()
+				if _, err := os.Stat(fn); err != nil {
+					t.Fatalf("Expected a key file at %q", fn)
+				}
+			}
 
-	// Stop current
-	s.Shutdown()
+			// Check stream meta.
+			checkEncrypted := func() {
+				checkKeyFile(filepath.Join(sdir, JetStreamMetaFileKey))
+				checkFor(filepath.Join(sdir, JetStreamMetaFile), "TEST", "foo", "bar", "baz", "max_msgs", "max_bytes")
+				// Check a message block.
+				checkKeyFile(filepath.Join(sdir, "msgs", "1.key"))
+				checkFor(filepath.Join(sdir, "msgs", "1.blk"), "ENCRYPTED PAYLOAD!!", "foo", "bar", "baz")
 
-	checkEncrypted()
+				// Check consumer meta and state.
+				checkKeyFile(filepath.Join(sdir, "obs", "dlc", JetStreamMetaFileKey))
+				checkFor(filepath.Join(sdir, "obs", "dlc", JetStreamMetaFile), "TEST", "dlc", "foo", "bar", "baz", "max_msgs", "ack_policy")
+				// Load and see if we can parse the consumer state.
+				state, err := os.ReadFile(filepath.Join(sdir, "obs", "dlc", "o.dat"))
+				if err != nil {
+					t.Fatalf("Unexpected error: %v", err)
+				}
+				if _, err := decodeConsumerState(state); err == nil {
+					t.Fatalf("Expected decoding consumer state to fail")
+				}
+			}
 
-	// Restart.
-	s, _ = RunServerWithConfig(conf)
-	defer s.Shutdown()
+			// Stop current
+			s.Shutdown()
 
-	// Connect again.
-	nc, js = jsClientConnect(t, s)
-	defer nc.Close()
+			checkEncrypted()
 
-	si2, err := js.StreamInfo("TEST")
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	if !reflect.DeepEqual(si, si2) {
-		t.Fatalf("Stream infos did not match\n%+v\nvs\n%+v", si, si2)
-	}
+			// Restart.
+			s, _ = RunServerWithConfig(conf)
+			defer s.Shutdown()
 
-	ci2, _ := js.ConsumerInfo("TEST", "dlc")
-	// Consumer create times can be slightly off after restore from disk.
-	now := time.Now()
-	ci.Created, ci2.Created = now, now
-	ci.Delivered.Last, ci2.Delivered.Last = nil, nil
-	ci.AckFloor.Last, ci2.AckFloor.Last = nil, nil
-	// Also clusters will be different.
-	ci.Cluster, ci2.Cluster = nil, nil
-	if !reflect.DeepEqual(ci, ci2) {
-		t.Fatalf("Consumer infos did not match\n%+v\nvs\n%+v", ci, ci2)
-	}
+			// Connect again.
+			nc, js = jsClientConnect(t, s)
+			defer nc.Close()
 
-	// Send 10 more msgs
-	for i := 0; i < 10; i++ {
-		sendMsg("foo")
-	}
-	if si, err = js.StreamInfo("TEST"); err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	if si.State.Msgs != 20 {
-		t.Fatalf("Expected 20 msgs total, got %d", si.State.Msgs)
-	}
+			si2, err := js.StreamInfo("TEST")
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+			if !reflect.DeepEqual(si, si2) {
+				t.Fatalf("Stream infos did not match\n%+v\nvs\n%+v", si, si2)
+			}
 
-	// Now test snapshots etc.
-	acc := s.GlobalAccount()
-	mset, err := acc.lookupStream("TEST")
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	scfg := mset.config()
-	sr, err := mset.snapshot(5*time.Second, false, true)
-	if err != nil {
-		t.Fatalf("Error getting snapshot: %v", err)
-	}
-	snapshot, err := io.ReadAll(sr.Reader)
-	if err != nil {
-		t.Fatalf("Error reading snapshot")
-	}
+			ci2, _ := js.ConsumerInfo("TEST", "dlc")
+			// Consumer create times can be slightly off after restore from disk.
+			now := time.Now()
+			ci.Created, ci2.Created = now, now
+			ci.Delivered.Last, ci2.Delivered.Last = nil, nil
+			ci.AckFloor.Last, ci2.AckFloor.Last = nil, nil
+			// Also clusters will be different.
+			ci.Cluster, ci2.Cluster = nil, nil
+			if !reflect.DeepEqual(ci, ci2) {
+				t.Fatalf("Consumer infos did not match\n%+v\nvs\n%+v", ci, ci2)
+			}
 
-	// Run new server w/o encryption. Make sure we can restore properly (meaning encryption was stripped etc).
-	ns := RunBasicJetStreamServer()
-	defer ns.Shutdown()
+			// Send 10 more msgs
+			for i := 0; i < 10; i++ {
+				sendMsg("foo")
+			}
+			if si, err = js.StreamInfo("TEST"); err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+			if si.State.Msgs != 20 {
+				t.Fatalf("Expected 20 msgs total, got %d", si.State.Msgs)
+			}
 
-	nacc := ns.GlobalAccount()
-	r := bytes.NewReader(snapshot)
-	mset, err = nacc.RestoreStream(&scfg, r)
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	ss := mset.store.State()
-	if ss.Msgs != si.State.Msgs || ss.FirstSeq != si.State.FirstSeq || ss.LastSeq != si.State.LastSeq {
-		t.Fatalf("Stream states do not match: %+v vs %+v", ss, si.State)
-	}
+			// Now test snapshots etc.
+			acc := s.GlobalAccount()
+			mset, err := acc.lookupStream("TEST")
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+			scfg := mset.config()
+			sr, err := mset.snapshot(5*time.Second, false, true)
+			if err != nil {
+				t.Fatalf("Error getting snapshot: %v", err)
+			}
+			snapshot, err := io.ReadAll(sr.Reader)
+			if err != nil {
+				t.Fatalf("Error reading snapshot")
+			}
 
-	// Now restore to our encrypted server as well.
-	if err := js.DeleteStream("TEST"); err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
+			// Run new server w/o encryption. Make sure we can restore properly (meaning encryption was stripped etc).
+			ns := RunBasicJetStreamServer()
+			defer ns.Shutdown()
 
-	acc = s.GlobalAccount()
-	r.Reset(snapshot)
-	mset, err = acc.RestoreStream(&scfg, r)
-	if err != nil {
-		t.Fatalf("Unexpected error: %v", err)
-	}
-	ss = mset.store.State()
-	if ss.Msgs != si.State.Msgs || ss.FirstSeq != si.State.FirstSeq || ss.LastSeq != si.State.LastSeq {
-		t.Fatalf("Stream states do not match: %+v vs %+v", ss, si.State)
-	}
+			nacc := ns.GlobalAccount()
+			r := bytes.NewReader(snapshot)
+			mset, err = nacc.RestoreStream(&scfg, r)
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+			ss := mset.store.State()
+			if ss.Msgs != si.State.Msgs || ss.FirstSeq != si.State.FirstSeq || ss.LastSeq != si.State.LastSeq {
+				t.Fatalf("Stream states do not match: %+v vs %+v", ss, si.State)
+			}
+
+			// Now restore to our encrypted server as well.
+			if err := js.DeleteStream("TEST"); err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			acc = s.GlobalAccount()
+			r.Reset(snapshot)
+			mset, err = acc.RestoreStream(&scfg, r)
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+			ss = mset.store.State()
+			if ss.Msgs != si.State.Msgs || ss.FirstSeq != si.State.FirstSeq || ss.LastSeq != si.State.LastSeq {
+				t.Fatalf("Stream states do not match: %+v vs %+v", ss, si.State)
+			}
 
-	// Check that all is encrypted like above since we know we need to convert since snapshots always plaintext.
-	checkEncrypted()
+			// Check that all is encrypted like above since we know we need to convert since snapshots always plaintext.
+			checkEncrypted()
+		})
+	}
 }
 
 // User report of bug.
@@ -18950,3 +18965,84 @@ func TestJetStreamProperErrorDueToOverlapSubjects(t *testing.T) {
 	t.Run("standalone", func(t *testing.T) { test(t, s) })
 	t.Run("clustered", func(t *testing.T) { test(t, c.randomServer()) })
 }
+
+func TestJetStreamServerCipherConvert(t *testing.T) {
+	tmpl := `
+		listen: 127.0.0.1:-1
+		jetstream: {key: s3cr3t, store_dir: '%s', cipher: %s}
+	`
+	storeDir := createDir(t, JetStreamStoreDir)
+	defer removeDir(t, storeDir)
+
+	// Create a stream and a consumer under one cipher, and restart the server with a new cipher.
+	conf := createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, "AES")))
+	defer removeFile(t, conf)
+
+	s, _ := RunServerWithConfig(conf)
+	defer s.Shutdown()
+
+	// Client based API
+	nc, js := jsClientConnect(t, s)
+	defer nc.Close()
+
+	cfg := &nats.StreamConfig{
+		Name:     "TEST",
+		Subjects: []string{"foo"},
+	}
+	if _, err := js.AddStream(cfg); err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	for i := 0; i < 1000; i++ {
+		msg := []byte(fmt.Sprintf("TOP SECRET DOCUMENT #%d", i+1))
+		_, err := js.Publish("foo", msg)
+		require_NoError(t, err)
+	}
+
+	// Make sure consumers convert as well.
+	sub, err := js.PullSubscribe("foo", "dlc")
+	require_NoError(t, err)
+	for _, m := range fetchMsgs(t, sub, 100, 5*time.Second) {
+		m.AckSync()
+	}
+
+	si, err := js.StreamInfo("TEST")
+	require_NoError(t, err)
+
+	ci, err := js.ConsumerInfo("TEST", "dlc")
+	require_NoError(t, err)
+
+	// Stop current
+	s.Shutdown()
+
+	conf = createConfFile(t, []byte(fmt.Sprintf(tmpl, storeDir, "ChaCha")))
+	defer removeFile(t, conf)
+
+	s, _ = RunServerWithConfig(conf)
+	defer s.Shutdown()
+
+	nc, js = jsClientConnect(t, s)
+	defer nc.Close()
+
+	si2, err := js.StreamInfo("TEST")
+	require_NoError(t, err)
+
+	if !reflect.DeepEqual(si, si2) {
+		t.Fatalf("Stream infos did not match\n%+v\nvs\n%+v", si, si2)
+	}
+
+	ci2, err := js.ConsumerInfo("TEST", "dlc")
+	require_NoError(t, err)
+
+	// Consumer create times can be slightly off after restore from disk.
+	now := time.Now()
+	ci.Created, ci2.Created = now, now
+	ci.Delivered.Last, ci2.Delivered.Last = nil, nil
+	ci.AckFloor.Last, ci2.AckFloor.Last = nil, nil
+	// Also clusters will be different.
+	ci.Cluster, ci2.Cluster = nil, nil
+
+	if !reflect.DeepEqual(ci, ci2) {
+		t.Fatalf("Consumer infos did not match\n%+v\nvs\n%+v", ci, ci2)
+	}
+}