Repository: Zac-HD/hypofuzz
Pull request #83: Include parametrization in db key
Author: login: Liam-DeVoe
Created: 2025-04-25T23:31:18Z | Merged: 2025-04-28T03:42:52Z

We weren't doing this before, either because the manual invocation of pytest doesn't run other plugins, or because the hypothesis plugin adds the key at `pytest_runtest_call` and not at collection time.


Diff:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d8c412d5..0da953b2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -44,8 +44,10 @@ jobs:
         )
       )
     permissions:
-      # required to push to gh-pages branch
+      # for pushing to the gh-pages branch
       contents: write
+      # for creating a new commit status for the docs page
+      statuses: write
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -96,6 +98,21 @@ jobs:
         git config --local user.name "GitHub Action"
         git commit -m "build docs (base=${BASE})"
         git push
+    - name: Create dashboard preview link
+      if: github.event_name == 'pull_request'
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          await github.rest.repos.createCommitStatus({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            sha: context.sha,
+            state: 'success',
+            target_url: `https://hypofuzz.com/preview-${context.payload.pull_request.number}/example-dashboard/`,
+            description: 'Dashboard preview',
+            context: 'dashboard-preview'
+          });
 
   test:
     runs-on: ubuntu-latest
diff --git a/docs-src/changelog.md b/docs-src/changelog.md
index c72c71a3..f5cde4f2 100644
--- a/docs-src/changelog.md
+++ b/docs-src/changelog.md
@@ -2,6 +2,10 @@
 HypoFuzz uses [calendar-based versioning](https://calver.org/), with a
 `YY-MM-patch` format.
 
+## 25.04.9
+
+Also filter the coverage graph when filtering the test table, and fix failures for parametrized tests being written to the incorrect database key.
+
 ## 25.04.8
 
 Improve text alignment of the test table UI.
diff --git a/src/hypofuzz/__init__.py b/src/hypofuzz/__init__.py
index 23dd1832..8adc03bc 100644
--- a/src/hypofuzz/__init__.py
+++ b/src/hypofuzz/__init__.py
@@ -1,4 +1,4 @@
 """Adaptive fuzzing for property-based tests using Hypothesis."""
 
-__version__ = "25.04.8"
+__version__ = "25.04.9"
 __all__: list = []
diff --git a/src/hypofuzz/dashboard.py b/src/hypofuzz/dashboard.py
index b04114fe..7755cbc9 100644
--- a/src/hypofuzz/dashboard.py
+++ b/src/hypofuzz/dashboard.py
@@ -139,8 +139,10 @@ async def websocket(websocket: WebSocket) -> None:
     websockets.add(websocket)
 
     reports = {
-        node_id: linearize_reports(reports) for node_id, reports in REPORTS.items()
+        nodeid: linearize_reports(reports) for nodeid, reports in REPORTS.items()
     }
+    for nodeid, reports_ in reports.items():
+        assert all(nodeid == report.nodeid for report in reports_.reports)
     await websocket.initial(reports, METADATA)
 
     try:
diff --git a/src/hypofuzz/frontend/src/components/CoverageGraph.tsx b/src/hypofuzz/frontend/src/components/CoverageGraph.tsx
index 4c557c3a..3c56ce66 100644
--- a/src/hypofuzz/frontend/src/components/CoverageGraph.tsx
+++ b/src/hypofuzz/frontend/src/components/CoverageGraph.tsx
@@ -1,9 +1,11 @@
-import { useEffect, useRef, useState } from "react"
+import { useEffect, useRef, useState, useMemo } from "react"
 import * as d3 from "d3"
 import { Report } from "../types/dashboard"
 import { Toggle } from "./Toggle"
 import { useSetting } from "../hooks/useSetting"
 import { useNavigate } from "react-router-dom"
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"
+import { faSearch, faTimes } from "@fortawesome/free-solid-svg-icons"
 // import BoxSelect from "../assets/box-select.svg?react"
 
 const mousePosition = { x: 0, y: 0 }
@@ -16,6 +18,7 @@ if (typeof window !== "undefined") {
 
 interface Props {
   reports: Map<string, Report[]>
+  filterString?: string
 }
 
 // in pixels
@@ -41,13 +44,14 @@ class Graph {
   private eventListeners: Map<string, Array<() => void>> = new Map()
   private xAxis: d3.Selection<SVGGElement, unknown, null, undefined>
   private yAxis: d3.Selection<SVGGElement, unknown, null, undefined>
-  private color: d3.ScaleOrdinal<string, string>
+  private reportsColor: d3.ScaleOrdinal<string, string>
   private viewportX: d3.ScaleContinuousNumeric<number, number>
   private viewportY: d3.ScaleContinuousNumeric<number, number>
 
   constructor(
     svg: SVGSVGElement,
     reports: Map<string, Report[]>,
+    reportsColor: d3.ScaleOrdinal<string, string>,
     scaleSetting: string,
     axisSetting: string,
     navigate: (path: string) => void,
@@ -63,7 +67,7 @@ class Graph {
     this.width = svg.clientWidth - this.margin.left - this.margin.right
     this.height = 300 - this.margin.top - this.margin.bottom
 
-    this.color = d3.scaleOrdinal(d3.schemeCategory10).domain(reports.keys())
+    this.reportsColor = reportsColor
     const allReports = Array.from(reports.values()).flat()
 
     // symlog is like log but defined linearly in the range [0, 1].
@@ -235,7 +239,7 @@ class Graph {
       .append("g")
       .attr("transform", `translate(${this.width + 10},0)`)
 
-    this.color.domain().forEach((nodeid, i) => {
+    this.reportsColor.domain().forEach((nodeid, i) => {
       const legendItem = legend
         .append("g")
         .attr("transform", `translate(0,${i * 20})`)
@@ -263,7 +267,7 @@ class Graph {
         .attr("x2", 20)
         .attr("y1", 10)
         .attr("y2", 10)
-        .attr("stroke", this.color(nodeid))
+        .attr("stroke", this.reportsColor(nodeid))
 
       legendItem
         .append("text")
@@ -285,7 +289,7 @@ class Graph {
         .append("path")
         .datum(points)
         .attr("fill", "none")
-        .attr("stroke", this.color(nodeid))
+        .attr("stroke", this.reportsColor(nodeid))
         .attr("d", line)
         .attr("class", "coverage-line")
         .style("cursor", "pointer")
@@ -384,7 +388,7 @@ class Graph {
   }
 }
 
-export function CoverageGraph({ reports }: Props) {
+export function CoverageGraph({ reports, filterString = "" }: Props) {
   const svgRef = useRef<SVGSVGElement>(null)
   const [scaleSetting, setScaleSetting] = useSetting<string>(
     "graph_scale",
@@ -402,8 +406,22 @@ export function CoverageGraph({ reports }: Props) {
   const [boxSelectEnabled, setBoxSelectEnabled] = useState(false)
   const navigate = useNavigate()
 
+  const filteredReports = useMemo(() => {
+    if (!filterString) return reports
+    return new Map(
+      Array.from(reports.entries()).filter(([nodeid]) =>
+        nodeid.toLowerCase().includes(filterString.toLowerCase()),
+      ),
+    )
+  }, [reports, filterString])
+
+  // use the unfiltered reports as the domain so colors are stable across filtering.
+  const reportsColor = d3
+    .scaleOrdinal(d3.schemeCategory10)
+    .domain(reports.keys())
+
   useEffect(() => {
-    if (!svgRef.current || reports.size === 0) {
+    if (!svgRef.current) {
       return
     }
 
@@ -433,7 +451,8 @@ export function CoverageGraph({ reports }: Props) {
     d3.select(svgRef.current).selectAll("*").remove()
     const graph = new Graph(
       svgRef.current,
-      reports,
+      filteredReports,
+      reportsColor,
       scaleSetting,
       axisSetting,
       navigate,
@@ -465,7 +484,7 @@ export function CoverageGraph({ reports }: Props) {
       graph.cleanup()
     }
   }, [
-    reports,
+    filteredReports,
     scaleSetting,
     axisSetting,
     forceUpdate,
diff --git a/src/hypofuzz/frontend/src/components/Table.tsx b/src/hypofuzz/frontend/src/components/Table.tsx
index 3050aa99..28f8db54 100644
--- a/src/hypofuzz/frontend/src/components/Table.tsx
+++ b/src/hypofuzz/frontend/src/components/Table.tsx
@@ -18,6 +18,7 @@ interface TableProps<T> {
   data: T[]
   row: (item: T) => React.ReactNode[]
   getKey?: (item: T) => string | number
+  onFilterChange?: (filter: string) => void
 }
 
 enum SortOrder {
@@ -37,7 +38,13 @@ function textContent(node: React.ReactNode): string {
   return ""
 }
 
-export function Table<T>({ headers, data, row, getKey }: TableProps<T>) {
+export function Table<T>({
+  headers,
+  data,
+  row,
+  getKey,
+  onFilterChange,
+}: TableProps<T>) {
   const [sortColumn, setSortColumn] = useState<number | null>(null)
   const [sortDirection, setSortDirection] = useState<SortOrder>(SortOrder.ASC)
   const [filterString, setFilterString] = useState("")
@@ -85,6 +92,11 @@ export function Table<T>({ headers, data, row, getKey }: TableProps<T>) {
     }
   }
 
+  function doFilterChange(filter: string) {
+    setFilterString(filter)
+    onFilterChange?.(filter)
+  }
+
   return (
     <div className="table">
       {/* only show filter box if some rows are filterable */}
@@ -94,13 +106,13 @@ export function Table<T>({ headers, data, row, getKey }: TableProps<T>) {
             type="text"
             placeholder="Filter"
             value={filterString}
-            onChange={e => setFilterString(e.target.value)}
+            onChange={e => doFilterChange(e.target.value)}
             className="table__filter__input"
           />
           {filterString && (
             <span
               className="table__filter__clear"
-              onClick={() => setFilterString("")}
+              onClick={() => doFilterChange("")}
             >
               <FontAwesomeIcon icon={faTimes} />
             </span>
@@ -165,7 +177,10 @@ export function Table<T>({ headers, data, row, getKey }: TableProps<T>) {
           Showing {displayData.length} of {data.length} rows
           <span
             className="table__filter-status__clear"
-            onClick={() => setFilterString("")}
+            onClick={() => {
+              setFilterString("")
+              onFilterChange?.("")
+            }}
           >
             [clear]
           </span>
diff --git a/src/hypofuzz/frontend/src/components/TestTable.tsx b/src/hypofuzz/frontend/src/components/TestTable.tsx
index 628e1292..235f987b 100644
--- a/src/hypofuzz/frontend/src/components/TestTable.tsx
+++ b/src/hypofuzz/frontend/src/components/TestTable.tsx
@@ -16,6 +16,7 @@ import { Tooltip } from "./Tooltip"
 interface Props {
   reports: Map<string, Report[]>
   metadata: Map<string, Metadata>
+  onFilterChange?: (filter: string) => void
 }
 
 interface TestRow {
@@ -23,18 +24,17 @@ interface TestRow {
   metadata: Metadata
 }
 
-export function TestTable({ reports, metadata }: Props) {
+export function TestTable({ reports, metadata, onFilterChange }: Props) {
   const sortedTests = Array.from(reports.entries())
-    .filter(([_nodeid, reports]) => {
+    .filter(([nodeid, reports]) => {
       if (reports.length === 0) {
         return false
       }
-      const latest = reports[reports.length - 1]
-      return metadata.has(latest.nodeid)
+      return metadata.has(nodeid)
     })
-    .sortKey(([_nodeid, reports]) => {
+    .sortKey(([nodeid, reports]) => {
       const latest = reports[reports.length - 1]
-      return [getStatus(latest, metadata.get(latest.nodeid)!), latest.nodeid]
+      return [getStatus(latest, metadata.get(nodeid)!), nodeid]
     })
     .map(([nodeid, reports]) => ({ reports, metadata: metadata.get(nodeid)! }))
 
@@ -175,6 +175,7 @@ export function TestTable({ reports, metadata }: Props) {
         data={sortedTests}
         row={row}
         getKey={item => item.reports[item.reports.length - 1].database_key}
+        onFilterChange={onFilterChange}
       />
     </div>
   )
diff --git a/src/hypofuzz/frontend/src/pages/Tests.tsx b/src/hypofuzz/frontend/src/pages/Tests.tsx
index 90ef6728..179177d1 100644
--- a/src/hypofuzz/frontend/src/pages/Tests.tsx
+++ b/src/hypofuzz/frontend/src/pages/Tests.tsx
@@ -1,14 +1,20 @@
 import { TestTable } from "../components/TestTable"
 import { CoverageGraph } from "../components/CoverageGraph"
 import { useData } from "../context/DataProvider"
+import { useState } from "react"
 
 export function TestsPage() {
   const { reports, metadata } = useData()
+  const [filterString, setFilterString] = useState("")
 
   return (
     <div className="dashboard">
-      <CoverageGraph reports={reports} />
-      <TestTable reports={reports} metadata={metadata} />
+      <CoverageGraph reports={reports} filterString={filterString} />
+      <TestTable
+        reports={reports}
+        metadata={metadata}
+        onFilterChange={setFilterString}
+      />
     </div>
   )
 }
diff --git a/src/hypofuzz/frontend/src/styles.scss b/src/hypofuzz/frontend/src/styles.scss
index b13a5d5b..2710ef5e 100644
--- a/src/hypofuzz/frontend/src/styles.scss
+++ b/src/hypofuzz/frontend/src/styles.scss
@@ -32,14 +32,6 @@ select {
     box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
 }
 
-.coverage-graph {
-    margin: 20px;
-    padding: 20px;
-    background: white;
-    border-radius: 8px;
-    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
-}
-
 .table {
     &__table {
         border-collapse: collapse;
diff --git a/src/hypofuzz/interface.py b/src/hypofuzz/interface.py
index b9a9c708..a2cf99cc 100644
--- a/src/hypofuzz/interface.py
+++ b/src/hypofuzz/interface.py
@@ -154,6 +154,11 @@ def pytest_collection_finish(self, session: pytest.Session) -> None:
                     )
                     extra_kw = {"factory": StateMachineClass}
                 else:
+                    # our pytest plugin would normally add this. Necessary to
+                    # distinguish pytest.mark.parametrize.
+                    item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = (
+                        item.nodeid.encode()
+                    )
                     target = item.obj
                 fuzz = FuzzProcess.from_hypothesis_test(
                     target, nodeid=item.nodeid, extra_kw=extra_kw
diff --git a/tests/common.py b/tests/common.py
index 0430a86a..09f2b8a1 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -119,12 +119,12 @@ def fuzz(*, n=1, dashboard=False, test_path=None):
     ]
     if test_path is not None:
         args += ["--", str(test_path)]
-    process = subprocess.Popen(args)
+    process = subprocess.Popen(args, start_new_session=True)
 
     try:
         yield
     finally:
-        process.terminate()
+        os.killpg(os.getpgid(process.pid), signal.SIGINT)
         process.wait()
 
 
diff --git a/tests/test_database.py b/tests/test_database.py
index a22fd36f..ddc0c02b 100644
--- a/tests/test_database.py
+++ b/tests/test_database.py
@@ -1,3 +1,5 @@
+import subprocess
+
 from common import BASIC_TEST_CODE, fuzz, wait_for, write_test_code
 from hypothesis.database import DirectoryBasedExampleDatabase
 
@@ -33,3 +35,39 @@ def test_database_stores_reports_and_metadata_correctly():
             )
             list(db.fetch_reports(key))
             list(db.fetch_metadata(key))
+
+
+def test_database_keys_incorporate_parametrization():
+    test_code = """
+        @pytest.mark.parametrize("x", [1, 2])
+        @given(st.integers())
+        def test_ints(x, n):
+            assert False
+        """
+    test_dir, db_dir = write_test_code(test_code)
+    db_hypofuzz = HypofuzzDatabase(DirectoryBasedExampleDatabase(db_dir))
+    assert not set(db_hypofuzz.fetch(b"hypofuzz-test-keys"))
+
+    with fuzz(test_path=test_dir, n=2):
+        # this will time out if the test keys are the same across parametrizations
+        wait_for(
+            lambda: len(set(db_hypofuzz.fetch(b"hypofuzz-test-keys"))) == 2,
+            timeout=10,
+            interval=0.1,
+        )
+
+    # we also test that the keys hypofuzz uses are the same as the ones hypothesis
+    # uses. We run pytest on the test file, which fails, causing hypothesis to write
+    # the failing choice sequence for each parametrization as a top-level key. These
+    # top-level keys should be the same as the hypofuzz keys.
+    test_dir, db_dir = write_test_code(test_code)
+    db_hypothesis = HypofuzzDatabase(DirectoryBasedExampleDatabase(db_dir))
+    hypofuzz_keys = set(db_hypofuzz.fetch(b"hypofuzz-test-keys"))
+    assert (
+        set(db_hypothesis.fetch(DirectoryBasedExampleDatabase._metakeys_name)) == set()
+    )
+
+    subprocess.run(["pytest", str(test_dir)], check=False)
+    assert hypofuzz_keys == set(
+        db_hypothesis.fetch(DirectoryBasedExampleDatabase._metakeys_name)
+    )


Reviews:
user: login: Zac-HD
state: COMMENTED
author_association: OWNER
submitted_at: 2025-04-26T22:10:42Z
user: login: Zac-HD
body: Looks good, but can we run the filter for both table and graph off a single input box?  I'd like to keep them in sync.  (then merge 😁)
state: APPROVED
author_association: OWNER
submitted_at: 2025-04-27T06:35:33Z

Review comments:
diff_hunk: @@ -33,3 +33,24 @@ def test_database_stores_reports_and_metadata_correctly():
             )
             list(db.fetch_reports(key))
             list(db.fetch_metadata(key))
+
+
+def test_database_keys_are_unique_by_nodeid():
+    test_dir, db_dir = write_test_code(
+        """
+        @pytest.mark.parametrize("x", [1, 2])
+        @given(st.integers())
+        def test_ints(x, n):
+            pass
+        """
+    )
+    db = HypofuzzDatabase(DirectoryBasedExampleDatabase(db_dir))
+    assert not list(db.fetch(b"hypofuzz-test-keys"))
+
+    with fuzz(test_path=test_dir, n=2):
+        # this will time out if the test keys are the same across parametrizations
+        wait_for(
+            lambda: len(list(db.fetch(b"hypofuzz-test-keys"))) == 2,
+            timeout=10,
+            interval=0.1,
+        )
path: tests/test_database.py
user: login: Zac-HD
body: I'd like to _also_ test that the test keys used by the fuzzer are the same two keys that would be used by running pytest+hypothesis over the same test code.
created_at: 2025-04-26T22:10:02Z
author_association: OWNER
diff_hunk: @@ -2,6 +2,10 @@
 HypoFuzz uses [calendar-based versioning](https://calver.org/), with a
 `YY-MM-patch` format.
 
+## 25.04.9
+
+Improve uniqueness of database keys for tests which use `@pytest.mark.parametrize`.
path: docs-src/changelog.md
user: login: Zac-HD
body: It's not just uniqueness; it's ensuring that Hypofuzz uses the same db keys as Hypothesis, so that replay works!
created_at: 2025-04-26T22:10:40Z
author_association: OWNER

PR comments:
user: login: Liam-DeVoe
created_at: 2025-04-26T00:56:00Z
body: direct link to the preview, pretty cool: <img width="1064" alt="image" src="https://github.com/user-attachments/assets/35402541-6a41-4315-b615-30ae4ed77148" />
author_association: COLLABORATOR
user: login: Liam-DeVoe
created_at: 2025-04-27T04:49:30Z
body: I've added a search bar to the coverage graph as well (just noticed: need to hide for the single-test case)
author_association: COLLABORATOR