Repository: nats-io/nats-server
Pull request #2109 (2021-04-13 01:11:58)

Change to report total deleted by default for stream info.
Allow deleted details if requested.

Clients owners may be impacted and will need to update clients. 
/cc @wallyqs @variadico @aricart @scottf 

Signed-off-by: Derek Collison <derek@nats.io>

/cc @nats-io/core



Patch:
diff --git a/server/filestore.go b/server/filestore.go
index af415cc1..3eb0ba49 100644
--- a/server/filestore.go
+++ b/server/filestore.go
@@ -2673,6 +2673,7 @@ func (fs *fileStore) State() StreamState {
 		sort.Slice(state.Deleted, func(i, j int) bool {
 			return state.Deleted[i] < state.Deleted[j]
 		})
+		state.NumDeleted = len(state.Deleted)
 	}
 	return state
 }
diff --git a/server/jetstream_api.go b/server/jetstream_api.go
index 7c1bc638..cd575533 100644
--- a/server/jetstream_api.go
+++ b/server/jetstream_api.go
@@ -306,6 +306,11 @@ type JSApiStreamDeleteResponse struct {
 
 const JSApiStreamDeleteResponseType = "io.nats.jetstream.api.v1.stream_delete_response"
 
+// JSApiStreamInfoRequest.
+type JSApiStreamInfoRequest struct {
+	DeletedDetails bool `json:"deleted_details,omitempty"`
+}
+
 // JSApiStreamInfoResponse.
 type JSApiStreamInfoResponse struct {
 	ApiResponse
@@ -1546,7 +1551,13 @@ func (s *Server) jsStreamListRequest(sub *subscription, c *client, subject, repl
 	}
 
 	for _, mset := range msets[offset:] {
-		resp.Streams = append(resp.Streams, &StreamInfo{Created: mset.createdTime(), State: mset.state(), Config: mset.config(), Mirror: mset.mirrorInfo(), Sources: mset.sourcesInfo()})
+		resp.Streams = append(resp.Streams, &StreamInfo{
+			Created: mset.createdTime(),
+			State:   mset.state(),
+			Config:  mset.config(),
+			Mirror:  mset.mirrorInfo(),
+			Sources: mset.sourcesInfo()},
+		)
 		if len(resp.Streams) >= JSApiListLimit {
 			break
 		}
@@ -1621,10 +1632,16 @@ func (s *Server) jsStreamInfoRequest(sub *subscription, c *client, subject, repl
 		s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
 		return
 	}
+
+	var details bool
 	if !isEmptyRequest(msg) {
-		resp.Error = jsNotEmptyRequestErr
-		s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
-		return
+		var req JSApiStreamInfoRequest
+		if err := json.Unmarshal(msg, &req); err != nil {
+			resp.Error = jsInvalidJSONErr
+			s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
+			return
+		}
+		details = req.DeletedDetails
 	}
 
 	mset, err := acc.lookupStream(streamName)
@@ -1637,7 +1654,12 @@ func (s *Server) jsStreamInfoRequest(sub *subscription, c *client, subject, repl
 
 	js, _ := s.getJetStreamCluster()
 
-	resp.StreamInfo = &StreamInfo{Created: mset.createdTime(), State: mset.state(), Config: config, Cluster: js.clusterInfo(mset.raftGroup())}
+	resp.StreamInfo = &StreamInfo{
+		Created: mset.createdTime(),
+		State:   mset.stateWithDetail(details),
+		Config:  config,
+		Cluster: js.clusterInfo(mset.raftGroup()),
+	}
 	if mset.isMirror() {
 		resp.StreamInfo.Mirror = mset.mirrorInfo()
 	} else if mset.hasSources() {
diff --git a/server/memstore.go b/server/memstore.go
index 20c0c75f..bd63a3a8 100644
--- a/server/memstore.go
+++ b/server/memstore.go
@@ -562,6 +562,7 @@ func (ms *memStore) State() StreamState {
 		sort.Slice(state.Deleted, func(i, j int) bool {
 			return state.Deleted[i] < state.Deleted[j]
 		})
+		state.NumDeleted = len(state.Deleted)
 	}
 	return state
 }
diff --git a/server/store.go b/server/store.go
index 69a54115..623d33a9 100644
--- a/server/store.go
+++ b/server/store.go
@@ -113,15 +113,16 @@ const (
 
 // StreamState is information about the given stream.
 type StreamState struct {
-	Msgs      uint64          `json:"messages"`
-	Bytes     uint64          `json:"bytes"`
-	FirstSeq  uint64          `json:"first_seq"`
-	FirstTime time.Time       `json:"first_ts"`
-	LastSeq   uint64          `json:"last_seq"`
-	LastTime  time.Time       `json:"last_ts"`
-	Deleted   []uint64        `json:"deleted,omitempty"`
-	Lost      *LostStreamData `json:"lost,omitempty"`
-	Consumers int             `json:"consumer_count"`
+	Msgs       uint64          `json:"messages"`
+	Bytes      uint64          `json:"bytes"`
+	FirstSeq   uint64          `json:"first_seq"`
+	FirstTime  time.Time       `json:"first_ts"`
+	LastSeq    uint64          `json:"last_seq"`
+	LastTime   time.Time       `json:"last_ts"`
+	NumDeleted int             `json:"num_deleted,omitempty"`
+	Deleted    []uint64        `json:"deleted,omitempty"`
+	Lost       *LostStreamData `json:"lost,omitempty"`
+	Consumers  int             `json:"consumer_count"`
 }
 
 // LostStreamData indicates msgs that have been lost.
diff --git a/server/stream.go b/server/stream.go
index 3296a56d..abfa941f 100644
--- a/server/stream.go
+++ b/server/stream.go
@@ -2914,6 +2914,10 @@ func (mset *stream) lookupConsumer(name string) *consumer {
 
 // State will return the current state for this stream.
 func (mset *stream) state() StreamState {
+	return mset.stateWithDetail(false)
+}
+
+func (mset *stream) stateWithDetail(details bool) StreamState {
 	mset.mu.RLock()
 	c, store := mset.client, mset.store
 	mset.mu.RUnlock()
@@ -2921,7 +2925,11 @@ func (mset *stream) state() StreamState {
 		return StreamState{}
 	}
 	// Currently rely on store.
-	return store.State()
+	state := store.State()
+	if !details {
+		state.Deleted = nil
+	}
+	return state
 }
 
 // Determines if the new proposed partition is unique amongst all consumers.


Test patch:
diff --git a/server/jetstream_cluster_test.go b/server/jetstream_cluster_test.go
index fdbf6bb2..919e3cd7 100644
--- a/server/jetstream_cluster_test.go
+++ b/server/jetstream_cluster_test.go
@@ -5442,7 +5442,6 @@ func TestJetStreamClusterAckPendingWithMaxRedelivered(t *testing.T) {
 
 	// Send in 100 messages.
 	msg, toSend := []byte("HELLO"), 100
-	rand.Read(msg)
 
 	for i := 0; i < toSend; i++ {
 		if _, err = js.Publish("foo", msg); err != nil {
@@ -5569,6 +5568,78 @@ func TestJetStreamClusterSuperClusterAndLeafNodesWithSharedSystemAccount(t *test
 	}
 }
 
+func TestJetStreamClusterStreamInfoDeletedDetails(t *testing.T) {
+	c := createJetStreamClusterExplicit(t, "R2", 2)
+	defer c.shutdown()
+
+	// Client based API
+	nc, js := jsClientConnect(t, c.randomServer())
+	defer nc.Close()
+
+	_, err := js.AddStream(&nats.StreamConfig{
+		Name:     "TEST",
+		Subjects: []string{"foo"},
+		Replicas: 1,
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	// Send in 10 messages.
+	msg, toSend := []byte("HELLO"), 10
+
+	for i := 0; i < toSend; i++ {
+		if _, err = js.Publish("foo", msg); err != nil {
+			t.Fatalf("Unexpected publish error: %v", err)
+		}
+	}
+
+	// Now remove some messages.
+	deleteMsg := func(seq uint64) {
+		if err := js.DeleteMsg("TEST", seq); err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+	}
+	deleteMsg(2)
+	deleteMsg(4)
+	deleteMsg(6)
+
+	// Need to do these via direct server request for now.
+	resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), nil, time.Second)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	var si StreamInfo
+	if err = json.Unmarshal(resp.Data, &si); err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if si.State.NumDeleted != 3 {
+		t.Fatalf("Expected %d deleted, got %d", 3, si.State.NumDeleted)
+	}
+	if len(si.State.Deleted) != 0 {
+		t.Fatalf("Expected not deleted details, but got %+v", si.State.Deleted)
+	}
+
+	// Now request deleted details.
+	req := JSApiStreamInfoRequest{DeletedDetails: true}
+	b, _ := json.Marshal(req)
+
+	resp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), b, time.Second)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if err = json.Unmarshal(resp.Data, &si); err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	if si.State.NumDeleted != 3 {
+		t.Fatalf("Expected %d deleted, got %d", 3, si.State.NumDeleted)
+	}
+	if len(si.State.Deleted) != 3 {
+		t.Fatalf("Expected deleted details, but got %+v", si.State.Deleted)
+	}
+}
+
 // Support functions
 
 // Used to setup superclusters for tests.
diff --git a/server/jetstream_test.go b/server/jetstream_test.go
index 2bf75a8c..ff37ba88 100644
--- a/server/jetstream_test.go
+++ b/server/jetstream_test.go
@@ -8274,9 +8274,11 @@ func TestJetStreamDeleteMsg(t *testing.T) {
 				}
 				expectedState.FirstTime = sm.Time
 				expectedState.Deleted = nil
+				expectedState.NumDeleted = 0
 
 				afterState := mset.state()
 				afterState.Deleted = nil
+				afterState.NumDeleted = 0
 
 				// Ignore first time in this test.
 				if !reflect.DeepEqual(afterState, expectedState) {
@@ -8326,7 +8328,7 @@ func TestJetStreamDeleteMsg(t *testing.T) {
 
 			expected := StreamState{Msgs: 6, Bytes: 6 * bytesPerMsg, FirstSeq: 12, LastSeq: 20}
 			state = mset.state()
-			state.FirstTime, state.LastTime, state.Deleted = time.Time{}, time.Time{}, nil
+			state.FirstTime, state.LastTime, state.Deleted, state.NumDeleted = time.Time{}, time.Time{}, nil, 0
 
 			if !reflect.DeepEqual(expected, state) {
 				t.Fatalf("State not what we expected. Expected %+v, got %+v\n", expected, state)