Repository: etcd-io/etcd
Pull request #19933: robustness: improved visualization for EtcdState and WatchOperation
Author: login: joshjms
Labels: name: area/testing
name: area/robustness-testing
name: size/M
name: approved
Created: 2025-05-13T09:52:00Z | Merged: 2025-05-13T11:05:06Z

Improved visualization for robustness test reports.
* Added `omitempty` to omit empty JSON fields
* Added indentation for report files (e.g. watch.json and operations.json)

Ref: https://github.com/etcd-io/etcd/issues/19901

/cc @serathius 

Please read https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#contribution-flow.



Diff:
diff --git a/tests/robustness/model/deterministic.go b/tests/robustness/model/deterministic.go
index 29f8d7f6361b..70aaa581959b 100644
--- a/tests/robustness/model/deterministic.go
+++ b/tests/robustness/model/deterministic.go
@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"hash/fnv"
+	"html"
 	"maps"
 	"reflect"
 	"sort"
@@ -54,20 +55,20 @@ var DeterministicModel = porcupine.Model{
 		return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), MaybeEtcdResponse{EtcdResponse: out.(EtcdResponse)}))
 	},
 	DescribeState: func(st any) string {
-		data, err := json.Marshal(st)
+		data, err := json.MarshalIndent(st, "", "  ")
 		if err != nil {
 			panic(err)
 		}
-		return string(data)
+		return "<pre>" + html.EscapeString(string(data)) + "</pre>"
 	},
 }
 
 type EtcdState struct {
-	Revision        int64
-	CompactRevision int64
-	KeyValues       map[string]ValueRevision
-	KeyLeases       map[string]int64
-	Leases          map[int64]EtcdLease
+	Revision        int64                    `json:",omitempty"`
+	CompactRevision int64                    `json:",omitempty"`
+	KeyValues       map[string]ValueRevision `json:",omitempty"`
+	KeyLeases       map[string]int64         `json:",omitempty"`
+	Leases          map[int64]EtcdLease      `json:",omitempty"`
 }
 
 func (s EtcdState) Equal(other EtcdState) bool {
@@ -458,14 +459,14 @@ func (el EtcdLease) DeepCopy() EtcdLease {
 }
 
 type ValueRevision struct {
-	Value       ValueOrHash
-	ModRevision int64
-	Version     int64
+	Value       ValueOrHash `json:",omitempty"`
+	ModRevision int64       `json:",omitempty"`
+	Version     int64       `json:",omitempty"`
 }
 
 type ValueOrHash struct {
-	Value string
-	Hash  uint32
+	Value string `json:",omitempty"`
+	Hash  uint32 `json:",omitempty"`
 }
 
 func ToValueOrHash(value string) ValueOrHash {
diff --git a/tests/robustness/model/non_deterministic.go b/tests/robustness/model/non_deterministic.go
index e48a815aa806..1bdbe87cd95e 100644
--- a/tests/robustness/model/non_deterministic.go
+++ b/tests/robustness/model/non_deterministic.go
@@ -17,6 +17,7 @@ package model
 import (
 	"encoding/json"
 	"fmt"
+	"html"
 	"reflect"
 
 	"github.com/anishathalye/porcupine"
@@ -40,11 +41,11 @@ var NonDeterministicModel = porcupine.Model{
 		return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(MaybeEtcdResponse)))
 	},
 	DescribeState: func(st any) string {
-		data, err := json.Marshal(st)
+		data, err := json.MarshalIndent(st, "", "  ")
 		if err != nil {
 			panic(err)
 		}
-		return string(data)
+		return "<pre>" + html.EscapeString(string(data)) + "</pre>"
 	},
 }
 
diff --git a/tests/robustness/model/replay.go b/tests/robustness/model/replay.go
index 7f2be26e8ed2..bed69d12f745 100644
--- a/tests/robustness/model/replay.go
+++ b/tests/robustness/model/replay.go
@@ -134,20 +134,20 @@ func toWatchEvents(prevState *EtcdState, request EtcdRequest, response MaybeEtcd
 }
 
 type WatchEvent struct {
-	PersistedEvent
-	PrevValue *ValueRevision
+	PersistedEvent `json:",omitempty"`
+	PrevValue      *ValueRevision `json:",omitempty"`
 }
 
 type PersistedEvent struct {
-	Event
-	Revision int64
-	IsCreate bool
+	Event    `json:",omitempty"`
+	Revision int64 `json:",omitempty"`
+	IsCreate bool  `json:",omitempty"`
 }
 
 type Event struct {
-	Type  OperationType
-	Key   string
-	Value ValueOrHash
+	Type  OperationType `json:",omitempty"`
+	Key   string        `json:",omitempty"`
+	Value ValueOrHash   `json:",omitempty"`
 }
 
 func (e Event) Match(request WatchRequest) bool {
diff --git a/tests/robustness/model/watch.go b/tests/robustness/model/watch.go
index fc880e30ede6..de2c54d1cc44 100644
--- a/tests/robustness/model/watch.go
+++ b/tests/robustness/model/watch.go
@@ -17,14 +17,14 @@ package model
 import "time"
 
 type WatchOperation struct {
-	Request   WatchRequest
-	Responses []WatchResponse
+	Request   WatchRequest    `json:",omitempty"`
+	Responses []WatchResponse `json:",omitempty"`
 }
 
 type WatchResponse struct {
-	Events           []WatchEvent
-	IsProgressNotify bool
-	Revision         int64
-	Time             time.Duration
-	Error            string
+	Events           []WatchEvent  `json:",omitempty"`
+	IsProgressNotify bool          `json:",omitempty"`
+	Revision         int64         `json:",omitempty"`
+	Time             time.Duration `json:",omitempty"`
+	Error            string        `json:",omitempty"`
 }
diff --git a/tests/robustness/report/client.go b/tests/robustness/report/client.go
index 208da9f0c1a0..9bfc7cbc6881 100644
--- a/tests/robustness/report/client.go
+++ b/tests/robustness/report/client.go
@@ -177,12 +177,13 @@ func persistWatchOperations(t *testing.T, lg *zap.Logger, path string, responses
 		return
 	}
 	defer file.Close()
-	encoder := json.NewEncoder(file)
 	for _, resp := range responses {
-		err := encoder.Encode(resp)
+		data, err := json.MarshalIndent(resp, "", "  ")
 		if err != nil {
 			t.Errorf("Failed to encode operation: %v", err)
 		}
+		file.Write(data)
+		file.WriteString("\n")
 	}
 }
 
@@ -194,11 +195,12 @@ func persistKeyValueOperations(t *testing.T, lg *zap.Logger, path string, operat
 		return
 	}
 	defer file.Close()
-	encoder := json.NewEncoder(file)
 	for _, op := range operations {
-		err := encoder.Encode(op)
+		data, err := json.MarshalIndent(op, "", "  ")
 		if err != nil {
 			t.Errorf("Failed to encode KV operation: %v", err)
 		}
+		file.Write(data)
+		file.WriteString("\n")
 	}
 }


Reviews:
user: login: serathius
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T10:58:44Z
user: login: serathius
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-13T10:59:40Z
user: login: serathius
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-13T11:00:03Z

Review comments:
diff_hunk: @@ -40,11 +41,11 @@ var NonDeterministicModel = porcupine.Model{
 		return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(MaybeEtcdResponse)))
 	},
 	DescribeState: func(st any) string {
-		data, err := json.Marshal(st)
+		data, err := json.MarshalIndent(st, "", "  ")
 		if err != nil {
 			panic(err)
 		}
-		return string(data)
+		return "<pre>" + html.EscapeString(string(data)) + "</pre>"
path: tests/robustness/model/non_deterministic.go
user: login: serathius
body: Oh, this is great! I didn't know you can make it work like this :P
created_at: 2025-05-13T10:58:44Z
author_association: MEMBER
diff_hunk: @@ -40,11 +41,11 @@ var NonDeterministicModel = porcupine.Model{
 		return fmt.Sprintf("%s -> %s", describeEtcdRequest(in.(EtcdRequest)), describeEtcdResponse(in.(EtcdRequest), out.(MaybeEtcdResponse)))
 	},
 	DescribeState: func(st any) string {
-		data, err := json.Marshal(st)
+		data, err := json.MarshalIndent(st, "", "  ")
 		if err != nil {
 			panic(err)
 		}
-		return string(data)
+		return "<pre>" + html.EscapeString(string(data)) + "</pre>"
path: tests/robustness/model/non_deterministic.go
user: login: serathius
body: I'm horrible on frontend :P
created_at: 2025-05-13T10:59:40Z
author_association: MEMBER

PR comments:
user: login: joshjms
created_at: 2025-05-13T10:01:09Z
body: To see the changes:

```bash
PERSIST_RESULTS=true EXPECT_DEBUG=true GO_TEST_FLAGS='-v' make test-robustness
...
logger.go:146: 2025-05-13T17:56:11.706+0800 INFO    Saving robustness test report   {"path": "/tmp/TestRobustnessRegression_Issue15271/1747130171705988186"}
...

cd /tmp/TestRobustnessRegression_Issue15271/1747130171705988186/client-1
cat watch.json
# Should show properly indented formatted JSON

cd /tmp/TestRobustnessRegression_Issue15271/1747130171705988186/client-2
cat operations.json
# Also should show properly indented formatted JSON
```

history.html:

![image](https://github.com/user-attachments/assets/0e7b62ed-c2c0-48fe-bef8-ae95f1438cde)

Unfortunately it's still too long vertically so it's difficult to read. Will raise an issue at porcupine as per @serathius ' recommendation.
author_association: MEMBER
user: login: codecov[bot]
created_at: 2025-05-13T10:16:21Z
body: ## [Codecov](https://app.codecov.io/gh/etcd-io/etcd/pull/19933?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 68.83%. Comparing base [(`276839a`)](https://app.codecov.io/gh/etcd-io/etcd/commit/276839a2e4663b4f30d3f3ab958f29a43cd98f7c?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io) to head [(`3546091`)](https://app.codecov.io/gh/etcd-io/etcd/commit/3546091fdf77be4bdbad134c75181d897fa78074?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io).
> Report is 4 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


[see 19 files with indirect coverage changes](https://app.codecov.io/gh/etcd-io/etcd/pull/19933/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io)

```diff
@@            Coverage Diff             @@
##             main   #19933      +/-   ##
==========================================
- Coverage   68.84%   68.83%   -0.02%     
==========================================
  Files         424      424              
  Lines       35878    35878              
==========================================
- Hits        24701    24697       -4     
- Misses       9751     9754       +3     
- Partials     1426     1427       +1     
```

------

[Continue to review full report in Codecov by Sentry](https://app.codecov.io/gh/etcd-io/etcd/pull/19933?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://app.codecov.io/gh/etcd-io/etcd/pull/19933?dropdown=coverage&src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io). Last update [276839a...3546091](https://app.codecov.io/gh/etcd-io/etcd/pull/19933?dropdown=coverage&src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=etcd-io).

</details>
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: k8s-ci-robot
created_at: 2025-05-13T11:00:07Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/etcd-io/etcd/pull/19933#" title="Author self-approved">joshjms</a>*, *<a href="https://github.com/etcd-io/etcd/pull/19933#pullrequestreview-2836267436" title="Approved">serathius</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=etcd-io%2Fetcd).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/etcd-io/etcd/blob/main/OWNERS)~~ [serathius]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: NONE
user: login: serathius
created_at: 2025-05-13T11:04:17Z
body: > Unfortunately it's still too long vertically so it's difficult to read. Will raise an issue at porcupine as per @serathius ' recommendation.

I think the problems we want to solve are:
* Well formatted JSON is more readable, but too verbose for the history. We would want to write our custom format similar to what we do in `DescribeOperation`. Like 
  ```
  State{
    "key1" -> "1", rev: 2, ver: 1`
    "key2" -> "2", rev: 3, ver: 1`
  }
  ```
* The state description is only visible on when hovering over a block. Adding a support to persist on click would be really helpfull. 
author_association: MEMBER