Repository: nats-io/nats-server
Pull request #1655 (2020-10-20 06:19:34)

Adding support and an option for removal of jwt
To enable in full mode configure allow_delete: true
When enabled the file will be renamed to allow for manual restore.

In cache mode it will be enabled by default.
When enabled files will be deleted.

For now deletes need to be done using the operator identity key.
Going forward we can allow signing keys to remove accounts issued by them while the operator identity key can be used to delete any jwt.

Signed-off-by: Matthias Hanel <mh@synadia.com>


Patch:
diff --git a/server/accounts.go b/server/accounts.go
index 10186371..11c5d6a0 100644
--- a/server/accounts.go
+++ b/server/accounts.go
@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/nats-io/jwt/v2"
+	"github.com/nats-io/nkeys"
 	"github.com/nats-io/nuid"
 )
 
@@ -3301,14 +3302,49 @@ func handleListRequest(store *DirJWTStore, s *Server, reply string) {
 }
 
 func handleDeleteRequest(store *DirJWTStore, s *Server, msg []byte, reply string) {
-	accIds := strings.Split(strings.Replace(string(msg), "\r\n", "\n", -1), "\n")
+	var accIds []interface{}
+	var subj, sysAccName string
+	if sysAcc := s.SystemAccount(); sysAcc != nil {
+		sysAccName = sysAcc.GetName()
+	}
+	// TODO Can allow keys (issuer) to delete accounts they issued and operator key to delete all accounts.
+	//      For now only operator is allowed to delete
+	gk, err := jwt.DecodeGeneric(string(msg))
+	if err == nil {
+		subj = gk.Subject
+		if store.deleteType == NoDelete {
+			err = fmt.Errorf("delete must be enabled in server config")
+		} else if subj != gk.Issuer {
+			err = fmt.Errorf("not self signed")
+		} else if !s.isTrustedIssuer(gk.Issuer) {
+			err = fmt.Errorf("not trusted")
+		} else if store.operator != gk.Issuer {
+			err = fmt.Errorf("needs to be the operator operator")
+		} else if list, ok := gk.Data["accounts"]; !ok {
+			err = fmt.Errorf("malformed request")
+		} else if accIds, ok = list.([]interface{}); !ok {
+			err = fmt.Errorf("malformed request")
+		} else {
+			for _, entry := range accIds {
+				if acc, ok := entry.(string); !ok ||
+					acc == "" || !nkeys.IsValidPublicAccountKey(acc) {
+					err = fmt.Errorf("malformed request")
+					break
+				} else if acc == sysAccName {
+					err = fmt.Errorf("not allowed to delete system account")
+					break
+				}
+			}
+		}
+	}
+	if err != nil {
+		respondToUpdate(s, reply, "", fmt.Sprintf("delete accounts request by %s failed", subj), err)
+		return
+	}
 	errs := []string{}
 	passCnt := 0
 	for _, acc := range accIds {
-		if acc == "" {
-			continue
-		}
-		if err := store.delete(acc); err != nil {
+		if err := store.delete(acc.(string)); err != nil {
 			errs = append(errs, err.Error())
 		} else {
 			passCnt++
@@ -3322,10 +3358,26 @@ func handleDeleteRequest(store *DirJWTStore, s *Server, msg []byte, reply string
 	}
 }
 
+func getOperator(s *Server) (string, error) {
+	var op string
+	if opts := s.getOpts(); opts != nil && len(opts.TrustedOperators) > 0 {
+		op = opts.TrustedOperators[0].Subject
+	}
+	if op == "" {
+		return "", fmt.Errorf("no operator found")
+	}
+	return op, nil
+}
+
 func (dr *DirAccResolver) Start(s *Server) error {
+	op, err := getOperator(s)
+	if err != nil {
+		return err
+	}
 	dr.Lock()
 	defer dr.Unlock()
 	dr.Server = s
+	dr.operator = op
 	dr.DirJWTStore.changed = func(pubKey string) {
 		if v, ok := s.accounts.Load(pubKey); !ok {
 		} else if jwt, err := dr.LoadAcc(pubKey); err != nil {
@@ -3476,14 +3528,18 @@ func (dr *DirAccResolver) Store(name, jwt string) error {
 	return dr.saveIfNewer(name, jwt)
 }
 
-func NewDirAccResolver(path string, limit int64, syncInterval time.Duration) (*DirAccResolver, error) {
+func NewDirAccResolver(path string, limit int64, syncInterval time.Duration, delete bool) (*DirAccResolver, error) {
 	if limit == 0 {
 		limit = math.MaxInt64
 	}
 	if syncInterval <= 0 {
 		syncInterval = time.Minute
 	}
-	store, err := NewExpiringDirJWTStore(path, false, true, 0, limit, false, 0, nil)
+	deleteType := NoDelete
+	if delete {
+		deleteType = RenameDeleted
+	}
+	store, err := NewExpiringDirJWTStore(path, false, true, deleteType, 0, limit, false, 0, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -3544,11 +3600,11 @@ func (s *Server) fetch(res AccountResolver, name string) (string, error) {
 	return theJWT, err
 }
 
-func NewCacheDirAccResolver(path string, limit int64, ttl time.Duration) (*CacheDirAccResolver, error) {
+func NewCacheDirAccResolver(path string, limit int64, ttl time.Duration, _ ...dirJWTStoreOption) (*CacheDirAccResolver, error) {
 	if limit <= 0 {
 		limit = 1_000
 	}
-	store, err := NewExpiringDirJWTStore(path, false, true, 0, limit, true, ttl, nil)
+	store, err := NewExpiringDirJWTStore(path, false, true, HardDelete, 0, limit, true, ttl, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -3556,9 +3612,14 @@ func NewCacheDirAccResolver(path string, limit int64, ttl time.Duration) (*Cache
 }
 
 func (dr *CacheDirAccResolver) Start(s *Server) error {
+	op, err := getOperator(s)
+	if err != nil {
+		return err
+	}
 	dr.Lock()
 	defer dr.Unlock()
 	dr.Server = s
+	dr.operator = op
 	dr.DirJWTStore.changed = func(pubKey string) {
 		if v, ok := s.accounts.Load(pubKey); !ok {
 		} else if jwt, err := dr.LoadAcc(pubKey); err != nil {
diff --git a/server/dirstore.go b/server/dirstore.go
index 76ddf295..5a6af63a 100644
--- a/server/dirstore.go
+++ b/server/dirstore.go
@@ -81,6 +81,8 @@ type DirJWTStore struct {
 	directory  string
 	shard      bool
 	readonly   bool
+	deleteType deleteType
+	operator   string
 	expiration *expirationTracker
 	changed    JWTChanged
 }
@@ -101,10 +103,13 @@ func newDir(dirPath string, create bool) (string, error) {
 	return fullPath, nil
 }
 
+// future proofing in case new options will be added
+type dirJWTStoreOption interface{}
+
 // Creates a directory based jwt store.
 // Reads files only, does NOT watch directories and files.
-func NewImmutableDirJWTStore(dirPath string, shard bool) (*DirJWTStore, error) {
-	theStore, err := NewDirJWTStore(dirPath, shard, false)
+func NewImmutableDirJWTStore(dirPath string, shard bool, _ ...dirJWTStoreOption) (*DirJWTStore, error) {
+	theStore, err := NewDirJWTStore(dirPath, shard, false, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -114,7 +119,7 @@ func NewImmutableDirJWTStore(dirPath string, shard bool) (*DirJWTStore, error) {
 
 // Creates a directory based jwt store.
 // Operates on files only, does NOT watch directories and files.
-func NewDirJWTStore(dirPath string, shard bool, create bool) (*DirJWTStore, error) {
+func NewDirJWTStore(dirPath string, shard bool, create bool, _ ...dirJWTStoreOption) (*DirJWTStore, error) {
 	fullPath, err := newDir(dirPath, create)
 	if err != nil {
 		return nil, err
@@ -126,6 +131,14 @@ func NewDirJWTStore(dirPath string, shard bool, create bool) (*DirJWTStore, erro
 	return theStore, nil
 }
 
+type deleteType int
+
+const (
+	NoDelete deleteType = iota
+	RenameDeleted
+	HardDelete
+)
+
 // Creates a directory based jwt store.
 //
 // When ttl is set deletion of file is based on it and not on the jwt expiration
@@ -135,16 +148,17 @@ func NewDirJWTStore(dirPath string, shard bool, create bool) (*DirJWTStore, erro
 // evictOnLimit determines the behavior once limit is reached.
 //     true - Evict based on lru strategy
 //     false - return an error
-func NewExpiringDirJWTStore(dirPath string, shard bool, create bool, expireCheck time.Duration, limit int64,
-	evictOnLimit bool, ttl time.Duration, changeNotification JWTChanged) (*DirJWTStore, error) {
+func NewExpiringDirJWTStore(dirPath string, shard bool, create bool, delete deleteType, expireCheck time.Duration, limit int64,
+	evictOnLimit bool, ttl time.Duration, changeNotification JWTChanged, _ ...dirJWTStoreOption) (*DirJWTStore, error) {
 	fullPath, err := newDir(dirPath, create)
 	if err != nil {
 		return nil, err
 	}
 	theStore := &DirJWTStore{
-		directory: fullPath,
-		shard:     shard,
-		changed:   changeNotification,
+		directory:  fullPath,
+		shard:      shard,
+		deleteType: delete,
+		changed:    changeNotification,
 	}
 	if expireCheck <= 0 {
 		if ttl != 0 {
@@ -423,17 +437,27 @@ func (store *DirJWTStore) write(path string, publicKey string, theJWT string) (b
 func (store *DirJWTStore) delete(publicKey string) error {
 	if store.readonly {
 		return fmt.Errorf("store is read-only")
+	} else if store.deleteType == NoDelete {
+		return fmt.Errorf("store is not set up to for delete")
 	}
 	store.Lock()
 	defer store.Unlock()
-	if err := os.Remove(store.pathForKey(publicKey)); err != nil {
-		if _, ok := err.(*os.PathError); ok || err == os.ErrNotExist {
+	name := store.pathForKey(publicKey)
+	if store.deleteType == RenameDeleted {
+		if err := os.Rename(name, name+".deleted"); err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+	} else if err := os.Remove(name); err != nil {
+		if os.IsNotExist(err) {
 			return nil
 		}
 		return err
 	}
 	store.expiration.unTrack(publicKey)
-	// TODO do cb
+	// TODO do cb so server can evict the account and associated clients
 	return nil
 }
 
diff --git a/server/opts.go b/server/opts.go
index 71b905fd..1dfbd054 100644
--- a/server/opts.go
+++ b/server/opts.go
@@ -849,6 +849,7 @@ func (o *Options) processConfigFileLine(k string, v interface{}, errors *[]error
 				}
 			}
 		case map[string]interface{}:
+			del := false
 			dir := ""
 			dirType := ""
 			limit := int64(0)
@@ -863,6 +864,10 @@ func (o *Options) processConfigFileLine(k string, v interface{}, errors *[]error
 				_, v := unwrapValue(v, &lt)
 				dirType = v.(string)
 			}
+			if v, ok := v["allow_delete"]; ok {
+				_, v := unwrapValue(v, &lt)
+				del = v.(bool)
+			}
 			if v, ok := v["limit"]; ok {
 				_, v := unwrapValue(v, &lt)
 				limit = v.(int64)
@@ -893,12 +898,15 @@ func (o *Options) processConfigFileLine(k string, v interface{}, errors *[]error
 				if sync != 0 {
 					*errors = append(*errors, &configErr{tk, "CACHE does not accept sync"})
 				}
+				if del {
+					*errors = append(*errors, &configErr{tk, "CACHE does not accept allow_delete"})
+				}
 				res, err = NewCacheDirAccResolver(dir, limit, ttl)
 			case "FULL":
 				if ttl != 0 {
 					*errors = append(*errors, &configErr{tk, "FULL does not accept ttl"})
 				}
-				res, err = NewDirAccResolver(dir, limit, sync)
+				res, err = NewDirAccResolver(dir, limit, sync, del)
 			}
 			if err != nil {
 				*errors = append(*errors, &configErr{tk, err.Error()})


Test patch:
diff --git a/server/dirstore_test.go b/server/dirstore_test.go
index 9d13e6dc..c5bdc720 100644
--- a/server/dirstore_test.go
+++ b/server/dirstore_test.go
@@ -426,7 +426,7 @@ func TestExpiration(t *testing.T) {
 	dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 	require_NoError(t, err)
 
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, time.Millisecond*100, 10, true, 0, nil)
+	dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 10, true, 0, nil)
 	require_NoError(t, err)
 	defer dirStore.Close()
 
@@ -464,7 +464,7 @@ func TestLimit(t *testing.T) {
 	dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 	require_NoError(t, err)
 
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, time.Millisecond*100, 5, true, 0, nil)
+	dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 5, true, 0, nil)
 	require_NoError(t, err)
 	defer dirStore.Close()
 
@@ -506,7 +506,7 @@ func TestLimitNoEvict(t *testing.T) {
 	dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 	require_NoError(t, err)
 
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, time.Millisecond*100, 2, false, 0, nil)
+	dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, false, 0, nil)
 	require_NoError(t, err)
 	defer dirStore.Close()
 
@@ -557,7 +557,7 @@ func TestLimitNoEvict(t *testing.T) {
 func TestLruLoad(t *testing.T) {
 	dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 	require_NoError(t, err)
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, time.Millisecond*100, 2, true, 0, nil)
+	dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, nil)
 	require_NoError(t, err)
 	defer dirStore.Close()
 
@@ -590,7 +590,7 @@ func TestLru(t *testing.T) {
 	dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 	require_NoError(t, err)
 
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, time.Millisecond*100, 2, true, 0, nil)
+	dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, nil)
 	require_NoError(t, err)
 	defer dirStore.Close()
 
@@ -638,7 +638,7 @@ func TestReload(t *testing.T) {
 	dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 	require_NoError(t, err)
 	notificationChan := make(chan struct{}, 5)
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, time.Millisecond*100, 2, true, 0, func(publicKey string) {
+	dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 2, true, 0, func(publicKey string) {
 		notificationChan <- struct{}{}
 	})
 	require_NoError(t, err)
@@ -699,7 +699,7 @@ func TestExpirationUpdate(t *testing.T) {
 	dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 	require_NoError(t, err)
 
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, time.Millisecond*100, 10, true, 0, nil)
+	dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, time.Millisecond*100, 10, true, 0, nil)
 	require_NoError(t, err)
 	defer dirStore.Close()
 
@@ -762,7 +762,7 @@ func TestTTL(t *testing.T) {
 		require_NoError(t, err)
 		require_Len(t, len(f), 1)
 	}
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, 100*time.Millisecond, 10, true, time.Second, nil)
+	dirStore, err := NewExpiringDirJWTStore(dir, false, false, NoDelete, 100*time.Millisecond, 10, true, time.Second, nil)
 	require_NoError(t, err)
 	defer dirStore.Close()
 
@@ -794,28 +794,41 @@ func TestTTL(t *testing.T) {
 }
 
 func TestRemove(t *testing.T) {
-	dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
-	require_NoError(t, err)
-	require_OneJWT := func() {
-		t.Helper()
-		f, err := ioutil.ReadDir(dir)
-		require_NoError(t, err)
-		require_Len(t, len(f), 1)
+	for deleteType, test := range map[deleteType]struct {
+		expected int
+		moved    int
+	}{
+		HardDelete:    {0, 0},
+		RenameDeleted: {0, 1},
+		NoDelete:      {1, 0},
+	} {
+		t.Run("", func(t *testing.T) {
+			dir, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
+			require_NoError(t, err)
+			require_OneJWT := func() {
+				t.Helper()
+				f, err := ioutil.ReadDir(dir)
+				require_NoError(t, err)
+				require_Len(t, len(f), 1)
+			}
+			dirStore, err := NewExpiringDirJWTStore(dir, false, false, deleteType, 0, 10, true, 0, nil)
+			require_NoError(t, err)
+			defer dirStore.Close()
+			accountKey, err := nkeys.CreateAccount()
+			require_NoError(t, err)
+			pubKey, err := accountKey.PublicKey()
+			require_NoError(t, err)
+			createTestAccount(t, dirStore, 0, accountKey)
+			require_OneJWT()
+			dirStore.delete(pubKey)
+			f, err := filepath.Glob(dir + string(os.PathSeparator) + "/*.jwt")
+			require_NoError(t, err)
+			require_Len(t, len(f), test.expected)
+			f, err = filepath.Glob(dir + string(os.PathSeparator) + "/*.jwt.deleted")
+			require_NoError(t, err)
+			require_Len(t, len(f), test.moved)
+		})
 	}
-	dirStore, err := NewExpiringDirJWTStore(dir, false, false, 0, 10, true, 0, nil)
-	require_NoError(t, err)
-	defer dirStore.Close()
-
-	accountKey, err := nkeys.CreateAccount()
-	require_NoError(t, err)
-	pubKey, err := accountKey.PublicKey()
-	require_NoError(t, err)
-	createTestAccount(t, dirStore, 0, accountKey)
-	require_OneJWT()
-	dirStore.delete(pubKey)
-	f, err := ioutil.ReadDir(dir)
-	require_NoError(t, err)
-	require_Len(t, len(f), 0)
 }
 
 const infDur = time.Duration(math.MaxInt64)
@@ -836,7 +849,7 @@ func TestNotificationOnPack(t *testing.T) {
 	}
 	dirPack, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 	require_NoError(t, err)
-	packStore, err := NewExpiringDirJWTStore(dirPack, false, false, infDur, 0, true, 0, notification)
+	packStore, err := NewExpiringDirJWTStore(dirPack, false, false, NoDelete, infDur, 0, true, 0, notification)
 	require_NoError(t, err)
 	// prefill the store with data
 	for k, v := range jwts {
@@ -852,7 +865,7 @@ func TestNotificationOnPack(t *testing.T) {
 	for _, shard := range []bool{true, false, true, false} {
 		dirMerge, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 		require_NoError(t, err)
-		mergeStore, err := NewExpiringDirJWTStore(dirMerge, shard, false, infDur, 0, true, 0, notification)
+		mergeStore, err := NewExpiringDirJWTStore(dirMerge, shard, false, NoDelete, infDur, 0, true, 0, notification)
 		require_NoError(t, err)
 		// set
 		err = mergeStore.Merge(msg)
@@ -892,7 +905,7 @@ func TestNotificationOnPackWalk(t *testing.T) {
 	for i := 0; i < storeCnt; i++ {
 		dirMerge, err := ioutil.TempDir(os.TempDir(), "jwtstore_test")
 		require_NoError(t, err)
-		mergeStore, err := NewExpiringDirJWTStore(dirMerge, true, false, infDur, 0, true, 0, nil)
+		mergeStore, err := NewExpiringDirJWTStore(dirMerge, true, false, NoDelete, infDur, 0, true, 0, nil)
 		require_NoError(t, err)
 		store[i] = mergeStore
 	}
diff --git a/server/jwt_test.go b/server/jwt_test.go
index 68591d06..0355de4e 100644
--- a/server/jwt_test.go
+++ b/server/jwt_test.go
@@ -4379,46 +4379,73 @@ func TestJWTAccountOps(t *testing.T) {
 		require_NoError(t, err)
 		*creds1 = genCredsFile(t, ujwt1, seed)
 	}
-	var syspub, sysjwt, sysCreds string
-	createAccountAndUser(&syspub, &sysjwt, &sysCreds)
-	var apub, ajwt1, aCreds1 string
-	createAccountAndUser(&apub, &ajwt1, &aCreds1)
-	defer os.Remove(sysCreds)
-	defer os.Remove(aCreds1)
-	dirSrv := createDir(t, "srv")
-	defer os.RemoveAll(dirSrv)
-	conf := createConfFile(t, []byte(fmt.Sprintf(`
+	generateRequest := func(accs []string) []byte {
+		t.Helper()
+		opk, _ := oKp.PublicKey()
+		c := jwt.NewGenericClaims(opk)
+		c.Data["accounts"] = accs
+		cJwt, err := c.Encode(oKp)
+		if err != nil {
+			t.Fatalf("Expected no error %v", err)
+		}
+		return []byte(cJwt)
+	}
+	for _, cfg := range []string{
+		`type: full
+ 		allow_delete: true`,
+		`type: cache`,
+	} {
+		t.Run("", func(t *testing.T) {
+			var syspub, sysjwt, sysCreds string
+			createAccountAndUser(&syspub, &sysjwt, &sysCreds)
+			var apub, ajwt1, aCreds1 string
+			createAccountAndUser(&apub, &ajwt1, &aCreds1)
+			defer os.Remove(sysCreds)
+			defer os.Remove(aCreds1)
+			dirSrv := createDir(t, "srv")
+			defer os.RemoveAll(dirSrv)
+			conf := createConfFile(t, []byte(fmt.Sprintf(`
 		listen: -1
 		operator: %s
 		system_account: %s
 		resolver: {
-			type: full
+			%s
 			dir: %s
 		}
-    `, ojwt, syspub, dirSrv)))
-	defer os.Remove(conf)
-	srv, _ := RunServerWithConfig(conf)
-	defer srv.Shutdown()
-	updateJwt(t, srv.ClientURL(), sysCreds, syspub, sysjwt, 1) // update system account jwt
-	updateJwt(t, srv.ClientURL(), sysCreds, apub, ajwt1, 1)    // set jwt
-	nc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(sysCreds))
-	defer nc.Close()
-	resp, err := nc.Request(accListReqSubj, nil, time.Second)
-	require_NoError(t, err)
-	require_True(t, strings.Contains(string(resp.Data), apub))
-	require_True(t, strings.Contains(string(resp.Data), syspub))
-	// delete nothing
-	resp, err = nc.Request(accDeleteReqSubj, nil, time.Second)
-	require_NoError(t, err)
-	require_True(t, strings.Contains(string(resp.Data), `"message": "deleted 0 accounts"`))
-	// issue delete, twice to also delete a non existing account
-	for i := 0; i < 2; i++ {
-		resp, err = nc.Request(accDeleteReqSubj, []byte(apub), time.Second)
-		require_NoError(t, err)
-		require_True(t, strings.Contains(string(resp.Data), `"message": "deleted 1 accounts"`))
-		resp, err = nc.Request(accListReqSubj, nil, time.Second)
-		require_False(t, strings.Contains(string(resp.Data), apub))
-		require_True(t, strings.Contains(string(resp.Data), syspub))
-		require_NoError(t, err)
+    `, ojwt, syspub, cfg, dirSrv)))
+			defer os.Remove(conf)
+			srv, _ := RunServerWithConfig(conf)
+			defer srv.Shutdown()
+			updateJwt(t, srv.ClientURL(), sysCreds, syspub, sysjwt, 1) // update system account jwt
+			// push jwt (for full resolver)
+			updateJwt(t, srv.ClientURL(), sysCreds, apub, ajwt1, 1) // set jwt
+			nc := natsConnect(t, srv.ClientURL(), nats.UserCredentials(sysCreds))
+			defer nc.Close()
+			// simulate nas resolver in case of a lookup request (cache)
+			nc.Subscribe(fmt.Sprintf(accLookupReqSubj, apub), func(msg *nats.Msg) {
+				msg.Respond([]byte(ajwt1))
+			})
+			// connect so there is a reason to cache the request
+			ncA := natsConnect(t, srv.ClientURL(), nats.UserCredentials(aCreds1))
+			ncA.Close()
+			resp, err := nc.Request(accListReqSubj, nil, time.Second)
+			require_NoError(t, err)
+			require_True(t, strings.Contains(string(resp.Data), apub))
+			require_True(t, strings.Contains(string(resp.Data), syspub))
+			// delete nothing
+			resp, err = nc.Request(accDeleteReqSubj, generateRequest([]string{}), time.Second)
+			require_NoError(t, err)
+			require_True(t, strings.Contains(string(resp.Data), `"message": "deleted 0 accounts"`))
+			// issue delete, twice to also delete a non existing account
+			for i := 0; i < 2; i++ {
+				resp, err = nc.Request(accDeleteReqSubj, generateRequest([]string{apub}), time.Second)
+				require_NoError(t, err)
+				require_True(t, strings.Contains(string(resp.Data), `"message": "deleted 1 accounts"`))
+				resp, err = nc.Request(accListReqSubj, nil, time.Second)
+				require_False(t, strings.Contains(string(resp.Data), apub))
+				require_True(t, strings.Contains(string(resp.Data), syspub))
+				require_NoError(t, err)
+			}
+		})
 	}
 }