Repository: nats-io/nats-server

Issue:
jetstream for an expired account doesn't restart when the account is updated
### Observed behavior

If you have an operator mode setup, and an account has jetstream assets these assets will stop when the account expires (this behaviour is correct). However if the account is updated, the server doesn't restart the assets unless restarted (reload has no effect).



### Expected behavior

updating the account in the resolver should re-start the JetStream assets or at the very least a reload of the server should make that work.

### Server and client version

2.11.1
clients latest

### Host environment

_No response_

### Steps to reproduce

[test_js_expires.zip](https://github.com/user-attachments/files/19799224/test_js_expires.zip)

Unzip the test project, will need `nsc` and `nats` cli, and node  installed.
```bash
cd test_js_expires
npm install

# will setup an operator, an account called SYS, and an account called A
# The initial setup will have a normal non-expiring account which will expire later
# This command will start up a server with those assets, and create the stream/consumer
# that are used for the scripts
# the end of the script will print some useful urls, and nats cli commands
./env.sh

# in one terminal
node producer.js

# update the account so that it expires in 1m (you can wait for the producer to bail
# out with bad credentials
./expire.sh

# when dead you can use the `nats server account info A... --json | jq .expired` 
# command that was printed by the script to verify that the JWT is expired

nats server account info A... --json | jq .expired

# rejuvenate the account
./expire.sh
# validate that the JWT is NOT expired
nats server account info A... --json | jq .expired

# restarting the producer fails, jetstream not available (the bug here)
node producer.js


```

Discussion:
@aricart: This setup narrows down the issue for https://github.com/nats-io/nats-server/issues/6775

Solution patch:
diff --git a/server/accounts.go b/server/accounts.go
index 72f6a8c6..09656ebc 100644
--- a/server/accounts.go
+++ b/server/accounts.go
@@ -3148,7 +3148,9 @@ func (a *Account) expiredTimeout() {
 	// Collect the clients and expire them.
 	cs := a.getClients()
 	for _, c := range cs {
-		c.accountAuthExpired()
+		if !isInternalClient(c.kind) {
+			c.accountAuthExpired()
+		}
 	}
 }
 
diff --git a/server/client.go b/server/client.go
index c6e76a95..bc59b725 100644
--- a/server/client.go
+++ b/server/client.go
@@ -1991,20 +1991,26 @@ func (c *client) processErr(errStr string) {
 
 // Password pattern matcher.
 var passPat = regexp.MustCompile(`"?\s*pass\S*?"?\s*[:=]\s*"?(([^",\r\n}])*)`)
+var tokenPat = regexp.MustCompile(`"?\s*auth_token\S*?"?\s*[:=]\s*"?(([^",\r\n}])*)`)
 
-// removePassFromTrace removes any notion of passwords from trace
+// removeSecretsFromTrace removes any notion of passwords/tokens from trace
 // messages for logging.
-func removePassFromTrace(arg []byte) []byte {
-	if !bytes.Contains(arg, []byte(`pass`)) {
-		return arg
+func removeSecretsFromTrace(arg []byte) []byte {
+	buf := redact("pass", passPat, arg)
+	return redact("auth_token", tokenPat, buf)
+}
+
+func redact(name string, pat *regexp.Regexp, proto []byte) []byte {
+	if !bytes.Contains(proto, []byte(name)) {
+		return proto
 	}
 	// Take a copy of the connect proto just for the trace message.
 	var _arg [4096]byte
-	buf := append(_arg[:0], arg...)
+	buf := append(_arg[:0], proto...)
 
-	m := passPat.FindAllSubmatchIndex(buf, -1)
+	m := pat.FindAllSubmatchIndex(buf, -1)
 	if len(m) == 0 {
-		return arg
+		return proto
 	}
 
 	redactedPass := []byte("[REDACTED]")
@@ -2015,7 +2021,7 @@ func removePassFromTrace(arg []byte) []byte {
 		start := i[2]
 		end := i[3]
 
-		// Replace password substring.
+		// Replace value substring.
 		buf = append(buf[:start], append(redactedPass, buf[end:]...)...)
 		break
 	}
@@ -6112,7 +6118,7 @@ func (c *client) getRawAuthUser() string {
 	case c.opts.JWT != _EMPTY_:
 		return c.pubKey
 	case c.opts.Token != _EMPTY_:
-		return c.opts.Token
+		return "[REDACTED]"
 	default:
 		return _EMPTY_
 	}
@@ -6129,7 +6135,7 @@ func (c *client) getAuthUser() string {
 	case c.opts.JWT != _EMPTY_:
 		return fmt.Sprintf("JWT User %q", c.pubKey)
 	case c.opts.Token != _EMPTY_:
-		return fmt.Sprintf("Token %q", c.opts.Token)
+		return fmt.Sprintf("Token %q", "[REDACTED]")
 	default:
 		return `User "N/A"`
 	}
diff --git a/server/monitor.go b/server/monitor.go
index ab738534..8e15f318 100644
--- a/server/monitor.go
+++ b/server/monitor.go
@@ -2875,20 +2875,19 @@ type MetaClusterInfo struct {
 
 // JSInfo has detailed information on JetStream.
 type JSInfo struct {
-	ID       string          `json:"server_id"`
-	Now      time.Time       `json:"now"`
-	Disabled bool            `json:"disabled,omitempty"`
-	Config   JetStreamConfig `json:"config,omitempty"`
-	Limits   *JSLimitOpts    `json:"limits,omitempty"`
 	JetStreamStats
-	Streams   int              `json:"streams"`
-	Consumers int              `json:"consumers"`
-	Messages  uint64           `json:"messages"`
-	Bytes     uint64           `json:"bytes"`
-	Meta      *MetaClusterInfo `json:"meta_cluster,omitempty"`
-
-	// aggregate raft info
+	ID             string           `json:"server_id"`
+	Now            time.Time        `json:"now"`
+	Disabled       bool             `json:"disabled,omitempty"`
+	Config         JetStreamConfig  `json:"config,omitempty"`
+	Limits         *JSLimitOpts     `json:"limits,omitempty"`
+	Streams        int              `json:"streams"`
+	Consumers      int              `json:"consumers"`
+	Messages       uint64           `json:"messages"`
+	Bytes          uint64           `json:"bytes"`
+	Meta           *MetaClusterInfo `json:"meta_cluster,omitempty"`
 	AccountDetails []*AccountDetail `json:"account_details,omitempty"`
+	Total          int              `json:"total"`
 }
 
 func (s *Server) accountDetail(jsa *jsAccount, optStreams, optConsumers, optCfg, optRaft, optStreamLeader bool) *AccountDetail {
@@ -3026,7 +3025,7 @@ func (s *Server) Jsz(opts *JSzOptions) (*JSInfo, error) {
 	if opts.Offset < 0 {
 		opts.Offset = 0
 	}
-	if opts.Limit <= 0 {
+	if opts.Limit == 0 {
 		opts.Limit = 1024
 	}
 	if opts.Consumer {
@@ -3070,6 +3069,8 @@ func (s *Server) Jsz(opts *JSzOptions) (*JSInfo, error) {
 	}
 	js.mu.RUnlock()
 
+	jsi.Total = len(accounts)
+
 	if mg := js.getMetaGroup(); mg != nil {
 		if ci := s.raftNodeToClusterInfo(mg); ci != nil {
 			jsi.Meta = &MetaClusterInfo{Name: ci.Name, Leader: ci.Leader, Peer: getHash(ci.Leader), Size: mg.ClusterSize()}
@@ -3113,15 +3114,18 @@ func (s *Server) Jsz(opts *JSzOptions) (*JSInfo, error) {
 	if filterIdx >= 0 {
 		accounts = accounts[filterIdx : filterIdx+1]
 	} else if opts.Accounts {
-		// Sort by name for a consistent read (barring any concurrent changes)
-		slices.SortFunc(accounts, func(i, j *jsAccount) int { return cmp.Compare(i.acc().Name, j.acc().Name) })
 
-		// Offset larger than the number of accounts.
-		offset := min(opts.Offset, len(accounts))
-		accounts = accounts[offset:]
+		if opts.Limit > 0 {
+			// Sort by name for a consistent read (barring any concurrent changes)
+			slices.SortFunc(accounts, func(i, j *jsAccount) int { return cmp.Compare(i.acc().Name, j.acc().Name) })
 
-		limit := min(opts.Limit, len(accounts))
-		accounts = accounts[:limit]
+			// Offset larger than the number of accounts.
+			offset := min(opts.Offset, len(accounts))
+			accounts = accounts[offset:]
+
+			limit := min(opts.Limit, len(accounts))
+			accounts = accounts[:limit]
+		}
 	} else {
 		accounts = nil
 	}
diff --git a/server/parser.go b/server/parser.go
index ed3eaa15..3ba608d6 100644
--- a/server/parser.go
+++ b/server/parser.go
@@ -940,7 +940,7 @@ func (c *client) parse(buf []byte) error {
 					return err
 				}
 				if trace {
-					c.traceInOp("CONNECT", removePassFromTrace(arg))
+					c.traceInOp("CONNECT", removeSecretsFromTrace(arg))
 				}
 				if err := c.processConnect(arg); err != nil {
 					return err
diff --git a/server/stream.go b/server/stream.go
index a8f9d26f..65afec73 100644
--- a/server/stream.go
+++ b/server/stream.go
@@ -5200,7 +5200,8 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte,
 
 	// If subject delete markers are used, ensure message TTL is that at minimum.
 	// Otherwise, subject delete markers could be missed if one already exists for this subject.
-	if ttl > 0 && mset.cfg.SubjectDeleteMarkerTTL > 0 {
+	// MaxMsgsPer=1 is an exception, because we'll only ever have one message.
+	if ttl > 0 && mset.cfg.SubjectDeleteMarkerTTL > 0 && mset.cfg.MaxMsgsPer != 1 {
 		if minTtl := int64(mset.cfg.SubjectDeleteMarkerTTL.Seconds()); ttl < minTtl {
 			ttl = minTtl
 		}
diff --git a/util/nats-server-hardened.service b/util/nats-server-hardened.service
index 0a1ad182..693562e9 100644
--- a/util/nats-server-hardened.service
+++ b/util/nats-server-hardened.service
@@ -11,16 +11,20 @@ Type=simple
 EnvironmentFile=-/etc/default/nats-server
 ExecStart=/usr/sbin/nats-server -c /etc/nats-server.conf
 ExecReload=/bin/kill -s HUP $MAINPID
-ExecStop=/bin/kill -s SIGINT $MAINPID
+
+# The nats-server uses SIGUSR2 to trigger Lame Duck Mode (LDM) shutdown
+# https://docs.nats.io/running-a-nats-service/nats_admin/lame_duck_mode
+ExecStop=/bin/kill -s SIGUSR2  $MAINPID
 
 User=nats
 Group=nats
 
 Restart=always
 RestartSec=5
-# The nats-server uses SIGUSR2 to trigger using Lame Duck Mode (LDM) shutdown
-KillSignal=SIGUSR2
-# You might want to adjust TimeoutStopSec too.
+
+# This should be `lame_duck_duration` + some buffer to finish the shutdown.
+# By default, `lame_duck_duration` is 2 mins.
+TimeoutStopSec=150
 
 # Capacity Limits
 # JetStream requires 2 FDs open per stream.
diff --git a/util/nats-server.service b/util/nats-server.service
index c2956bb7..e4be9d3b 100644
--- a/util/nats-server.service
+++ b/util/nats-server.service
@@ -7,12 +7,17 @@ PrivateTmp=true
 Type=simple
 ExecStart=/usr/sbin/nats-server -c /etc/nats-server.conf
 ExecReload=/bin/kill -s HUP $MAINPID
-ExecStop=/bin/kill -s SIGINT $MAINPID
+
+# The nats-server uses SIGUSR2 to trigger Lame Duck Mode (LDM) shutdown
+# https://docs.nats.io/running-a-nats-service/nats_admin/lame_duck_mode
+ExecStop=/bin/kill -s SIGUSR2  $MAINPID
+
+# This should be `lame_duck_duration` + some buffer to finish the shutdown.
+# By default, `lame_duck_duration` is 2 mins.
+TimeoutStopSec=150
+
 User=nats
 Group=nats
-# The nats-server uses SIGUSR2 to trigger using Lame Duck Mode (LDM) shutdown
-KillSignal=SIGUSR2
-# You might want to adjust TimeoutStopSec too.
 
 [Install]
 WantedBy=multi-user.target



Test patch:
diff --git a/server/client_test.go b/server/client_test.go
index 1b37b48f..eb59248f 100644
--- a/server/client_test.go
+++ b/server/client_test.go
@@ -1624,6 +1624,18 @@ func TestClientUserInfo(t *testing.T) {
 	if got != expected {
 		t.Errorf("Expected %q, got %q", expected, got)
 	}
+
+	c = &client{
+		cid: 1024,
+		opts: ClientOpts{
+			Token: "s3cr3t!",
+		},
+	}
+	got = c.getAuthUser()
+	expected = `Token "[REDACTED]"`
+	if got != expected {
+		t.Errorf("Expected %q, got %q", expected, got)
+	}
 }
 
 type captureWarnLogger struct {
diff --git a/server/jetstream_cluster_4_test.go b/server/jetstream_cluster_4_test.go
index 1db0150d..e2d73d1d 100644
--- a/server/jetstream_cluster_4_test.go
+++ b/server/jetstream_cluster_4_test.go
@@ -5579,6 +5579,71 @@ func TestJetStreamClusterSubjectDeleteMarkersMinimumTTL(t *testing.T) {
 	}
 }
 
+func TestJetStreamClusterSubjectDeleteMarkersMinimumTTLExceptionMaxMsgsPer(t *testing.T) {
+	for _, storageType := range []StorageType{FileStorage, MemoryStorage} {
+		for _, replicas := range []int{1, 3} {
+			t.Run(fmt.Sprintf("%s/R%d", storageType, replicas), func(t *testing.T) {
+				c := createJetStreamClusterExplicit(t, "R3S", 3)
+				defer c.shutdown()
+
+				nc, js := jsClientConnect(t, c.randomServer())
+				defer nc.Close()
+
+				_, err := jsStreamCreate(t, nc, &StreamConfig{
+					Name:                   "TEST",
+					Retention:              LimitsPolicy,
+					Subjects:               []string{"foo"},
+					Replicas:               replicas,
+					Storage:                storageType,
+					SubjectDeleteMarkerTTL: time.Hour,
+					AllowMsgTTL:            true,
+					MaxMsgsPer:             1,
+				})
+				require_NoError(t, err)
+
+				m := nats.NewMsg("foo")
+				m.Header.Set(JSMessageTTL, "1s")
+				_, err = js.PublishMsg(m)
+				require_NoError(t, err)
+
+				_, err = js.GetMsg("TEST", 1)
+				require_NoError(t, err)
+
+				// After the TTL expires the message should be gone, even though SubjectDeleteMarkerTTL is the minimum
+				// normally. This works because MaxMsgsPer=1 is set.
+				time.Sleep(1500 * time.Millisecond)
+				_, err = js.GetMsg("TEST", 1)
+				require_Error(t, err, nats.ErrMsgNotFound)
+
+				// Expect a subject delete marker based on per-message TTL.
+				sm, err := js.GetMsg("TEST", 2)
+				require_NoError(t, err)
+				require_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)
+				require_Equal(t, sm.Subject, "foo")
+
+				// Since we have a subject delete marker now with a higher TTL, if this
+				// message's lower TTL would be respected then we would not have a new
+				// subject delete marker when this message expires. But with MaxMsgsPer=1
+				// this doesn't apply, because the subject delete marker gets removed.
+				pa, err := js.PublishMsg(m)
+				require_NoError(t, err)
+				require_Equal(t, pa.Sequence, 3)
+
+				// After some time the first subject delete marker should be gone, and the new one should be placed.
+				time.Sleep(1500 * time.Millisecond)
+
+				_, err = js.GetMsg("TEST", 2)
+				require_Error(t, err, nats.ErrMsgNotFound)
+
+				sm, err = js.GetMsg("TEST", 4)
+				require_NoError(t, err)
+				require_Equal(t, sm.Header.Get(JSMarkerReason), JSMarkerReasonMaxAge)
+				require_Equal(t, sm.Subject, "foo")
+			})
+		}
+	}
+}
+
 func TestJetStreamClusterSubjectDeleteMarkersNoMsgTTLSet(t *testing.T) {
 	for _, storageType := range []StorageType{FileStorage, MemoryStorage} {
 		for _, replicas := range []int{1, 3} {
diff --git a/server/jetstream_jwt_test.go b/server/jetstream_jwt_test.go
index 580b4054..6935eb15 100644
--- a/server/jetstream_jwt_test.go
+++ b/server/jetstream_jwt_test.go
@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -990,6 +991,124 @@ func TestJetStreamJWTExpiredAccountNotCountedTowardLimits(t *testing.T) {
 	require_True(t, ai.Limits.MaxMemory == 7*1024*1024)
 }
 
+func TestJetStreamJWTExpiredAccountWorksAfterExpirationUpdated(t *testing.T) {
+	sysKp, spub := createKey(t)
+	sysClaim := jwt.NewAccountClaims(spub)
+	sysClaim.Name = "$SYS"
+	sysJwt, err := sysClaim.Encode(oKp)
+	require_NoError(t, err)
+	sysCreds := newUser(t, sysKp)
+
+	akp, apub := createKey(t)
+	require_NoError(t, err)
+	accClaim := jwt.NewAccountClaims(apub)
+	accClaim.Name = "TEST"
+	accClaim.Limits.JetStreamLimits = jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1}
+	accJwt, err := accClaim.Encode(oKp)
+	require_NoError(t, err)
+
+	dirSrv := t.TempDir()
+	dir := t.TempDir()
+	conf := createConfFile(t, []byte(fmt.Sprintf(`
+				listen: 127.0.0.1:-1
+				jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb, store_dir: "%s"}
+				operator: %s
+				system_account: %s
+				resolver: {
+					type: full
+					allow_delete: true
+					dir: '%s'
+				}
+			`, dirSrv, ojwt, spub, dir)))
+	defer removeFile(t, conf)
+
+	s, _ := RunServerWithConfig(conf)
+	defer s.Shutdown()
+
+	updateJwt(t, s.ClientURL(), sysCreds, sysJwt, 1)
+	updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1)
+
+	userCreds := newUser(t, akp)
+	nc, js := jsClientConnect(t, s, nats.UserCredentials(userCreds), nats.NoReconnect(),
+		nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
+			// Default handler would print to stderr, silence it
+		}))
+	defer nc.Close()
+
+	_, err = js.AddStream(&nats.StreamConfig{
+		Name:     "TEST",
+		Subjects: []string{"foo"},
+	})
+	require_NoError(t, err)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for {
+			if _, err := js.Publish("foo", []byte("hello")); err != nil {
+				return
+			}
+			time.Sleep(250 * time.Millisecond)
+		}
+	}()
+
+	// Update the account so it expires in 2 seconds.
+	expires := time.Now().Add(2 * time.Second)
+	accClaim.Expires = expires.Unix()
+	accJwt = encodeClaim(t, accClaim, apub)
+	updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1)
+
+	// Wait for the publishing routine to fail.
+	wg.Wait()
+
+	// Close this client, we will create a new one.
+	nc.Close()
+
+	// Verify that we can't connect anymore. Because of rounding, we may have
+	// to try to avoid flapping.
+	for range 5 {
+		nc, err = nats.Connect(s.ClientURL(), nats.UserCredentials(userCreds), nats.NoReconnect(),
+			nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
+				// Default handler would print to stderr, silence it
+			}))
+		if err != nil {
+			// ok!
+			break
+		}
+		nc.Close()
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// Check that the user account is marked as expired.
+	az, err := s.Accountz(&AccountzOptions{Account: apub})
+	require_NoError(t, err)
+	require_True(t, az.Account != nil)
+	require_True(t, az.Account.Expired)
+	// But should still be true
+	require_True(t, az.Account.JetStream)
+
+	// Update the expiration to 1 hour
+	expires = time.Now().Add(time.Hour)
+	accClaim.Expires = expires.Unix()
+	accJwt = encodeClaim(t, accClaim, apub)
+	updateJwt(t, s.ClientURL(), sysCreds, accJwt, 1)
+
+	// Check that its is no longer expired and has still JetStream enabled.
+	az, err = s.Accountz(&AccountzOptions{Account: apub})
+	require_NoError(t, err)
+	require_True(t, az.Account != nil)
+	require_False(t, az.Account.Expired)
+	require_True(t, az.Account.JetStream)
+
+	// Create a connection and ensure we connect ok and can send a message.
+	nc, js = jsClientConnect(t, s, nats.UserCredentials(userCreds), nats.NoReconnect())
+	defer nc.Close()
+
+	_, err = js.Publish("foo", []byte("hello"))
+	require_NoError(t, err)
+}
+
 func TestJetStreamJWTDeletedAccountDoesNotLeakSubscriptions(t *testing.T) {
 	op, _ := nkeys.CreateOperator()
 	opPk, _ := op.PublicKey()
diff --git a/server/log_test.go b/server/log_test.go
index 57add588..55220489 100644
--- a/server/log_test.go
+++ b/server/log_test.go
@@ -341,7 +341,135 @@ func TestRemovePassFromTrace(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			output := removePassFromTrace([]byte(test.input))
+			output := removeSecretsFromTrace([]byte(test.input))
+			if !bytes.Equal(output, []byte(test.expected)) {
+				t.Errorf("\nExpected %q\n    got: %q", test.expected, string(output))
+			}
+		})
+	}
+}
+
+func TestRemoveAuthTokenFromTrace(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			"user and auth_token",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":\"s3cr3t\"}\r\n",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":\"[REDACTED]\"}\r\n",
+		},
+		{
+			"user and pass extra space",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":  \"s3cr3t\"}\r\n",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":  \"[REDACTED]\"}\r\n",
+		},
+		{
+			"user and pass is empty",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":\"\"}\r\n",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":\"[REDACTED]\"}\r\n",
+		},
+		{
+			"user and pass is empty whitespace",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":\"               \"}\r\n",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":\"[REDACTED]\"}\r\n",
+		},
+		{
+			"user and pass whitespace",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":    \"s3cr3t\"     }\r\n",
+			"CONNECT {\"user\":\"derek\",\"auth_token\":    \"[REDACTED]\"     }\r\n",
+		},
+		{
+			"only pass",
+			"CONNECT {\"auth_token\":\"s3cr3t\",}\r\n",
+			"CONNECT {\"auth_token\":\"[REDACTED]\",}\r\n",
+		},
+		{
+			"invalid json",
+			"CONNECT {auth_token:s3cr3t ,   password =  s3cr3t}",
+			"CONNECT {auth_token:[REDACTED],   password =  [REDACTED]}",
+		},
+		{
+			"invalid json no whitespace after key",
+			"CONNECT {auth_token:s3cr3t ,   password=  s3cr3t}",
+			"CONNECT {auth_token:[REDACTED],   password=  [REDACTED]}",
+		},
+		{
+			"both pass and wrong password key",
+			`CONNECT {"auth_token":"s3cr3t4", "password": "s3cr3t4"}`,
+			`CONNECT {"auth_token":"[REDACTED]", "password": "[REDACTED]"}`,
+		},
+		{
+			"invalid json",
+			"CONNECT {user = hello, auth_token =  s3cr3t}",
+			"CONNECT {user = hello, auth_token =  [REDACTED]}",
+		},
+		{
+			"complete connect",
+			"CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n",
+			"CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n",
+		},
+		{
+			"invalid json with only pass key",
+			"CONNECT {auth_token:s3cr3t\r\n",
+			"CONNECT {auth_token:[REDACTED]\r\n",
+		},
+		{
+			"invalid password key also filtered",
+			"CONNECT {\"auth_token\":\"s3cr3t\",}\r\n",
+			"CONNECT {\"auth_token\":\"[REDACTED]\",}\r\n",
+		},
+		{
+			"single long password with whitespace",
+			"CONNECT {\"auth_token\":\"secret password which is very long\",}\r\n",
+			"CONNECT {\"auth_token\":\"[REDACTED]\",}\r\n",
+		},
+		{
+			"single long pass key is filtered",
+			"CONNECT {\"auth_token\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"}\r\n",
+			"CONNECT {\"auth_token\":\"[REDACTED]\"}\r\n",
+		},
+		{
+			"duplicate keys only filtered once",
+			"CONNECT {\"auth_token\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\",\"auth_token\":\"BBB\",\"pass\":\"BBBBBBBBBBBBBBBBBBBB\",\"password\":\"CCCCCCCCCCCCCCCC\"}\r\n",
+			"CONNECT {\"auth_token\":\"[REDACTED]\",\"auth_token\":\"BBB\",\"pass\":\"[REDACTED]\",\"password\":\"CCCCCCCCCCCCCCCC\"}\r\n",
+		},
+		{
+			"invalid json with multiple keys only one is filtered",
+			"CONNECT {auth_token = \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\",pass= \"BBBBBBBBBBBBBBBBBBBB\",password =\"CCCCCCCCCCCCCCCC\"}\r\n",
+			"CONNECT {auth_token = \"[REDACTED]\",pass= \"[REDACTED]\",password =\"CCCCCCCCCCCCCCCC\"}\r\n",
+		},
+		{
+			"complete connect protocol",
+			"CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n",
+			"CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"foo\",\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"APM7JU94z77YzP6WTBEiuw\"}\r\n",
+		},
+		{
+			"user and token are filterered",
+			"CONNECT {\"user\":\"s3cr3t\",\"auth_token\":\"s3cr3t\"}\r\n",
+			"CONNECT {\"user\":\"s3cr3t\",\"auth_token\":\"[REDACTED]\"}\r\n",
+		},
+		{
+			"complete connect using token key with user and token being the same",
+			"CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"...\"}\r\n",
+			"CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"...\"}\r\n",
+		},
+		{
+			"complete connect with user, token and name all the same",
+			"CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"s3cr3t\"}\r\n",
+			"CONNECT {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"s3cr3t\"}\r\n",
+		},
+		{
+			"complete connect extra white space at the beginning",
+			"CONNECT 	 {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"s3cr3t\",\"tls_required\":false,\"name\":\"foo\"}\r\n",
+			"CONNECT 	 {\"echo\":true,\"verbose\":false,\"pedantic\":false,\"user\":\"s3cr3t\",\"auth_token\":\"[REDACTED]\",\"tls_required\":false,\"name\":\"foo\"}\r\n",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			output := removeSecretsFromTrace([]byte(test.input))
 			if !bytes.Equal(output, []byte(test.expected)) {
 				t.Errorf("\nExpected %q\n    got: %q", test.expected, string(output))
 			}
diff --git a/server/monitor_test.go b/server/monitor_test.go
index 86111cb2..2eedfb56 100644
--- a/server/monitor_test.go
+++ b/server/monitor_test.go
@@ -4752,7 +4752,7 @@ func TestMonitorAuthorizedUsers(t *testing.T) {
 		nats.Token("sometoken"))
 	defer c.Close()
 	// We should get the token specified by the user
-	checkAuthUser("sometoken")
+	checkAuthUser("[REDACTED]")
 	c.Close()
 	s.Shutdown()
 
@@ -4934,6 +4934,9 @@ func TestMonitorJsz(t *testing.T) {
 			if info.Limits.MaxHAAssets != 1000 {
 				t.Fatalf("expected max_ha_assets limit to be 1000 got %v", info.Limits)
 			}
+			if info.Total != 2 {
+				t.Fatalf("expected total to be 2 but got %d", info.Total)
+			}
 		}
 	})
 	t.Run("accounts", func(t *testing.T) {