Domain: catalog with derived-term search index

Issue:
Search keeps returning records for values they no longer have. After editing a record's tags or title, searches for the old tag or the old title words still list the record, alongside correct results for the new values. Deleting a record cleans up properly — only edits leave residue, and re-editing several times leaves residue for every past value. One operational note: our capacity dashboards track term derivations, which are expected to stay flat per edit at steady state — whatever the fix, routine editing must not turn into catalog-wide reindexing work.

Repository files:
content: # Present so pytest adds the repository root to sys.path, making `import src`
# resolve from the test suite.

path: conftest.py

content: # Pure standard library unless the environment image pre-bakes wheels.

path: requirements.lock

path: src/__init__.py

content: """The catalog: record storage plus its search index, kept in step."""

from .errors import UnknownRecord
from .index import TermIndex
from .records import Record
from .terms import derive_terms


class Catalog:
    def __init__(self):
        self._records = {}
        self.index = TermIndex()

    def __len__(self):
        return len(self._records)

    def get(self, record_id):
        try:
            return self._records[record_id]
        except KeyError:
            raise UnknownRecord(record_id) from None

    def add(self, record):
        self._records[record.record_id] = record
        for term in derive_terms(record):
            self.index.add(term, record.record_id)

    def load(self, records):
        """Bulk import: store everything, then build the index in one pass."""
        for record in records:
            self._records[record.record_id] = record
        self.index.rebuild(self._records.values())

    def update(self, record_id, title=None, tags=None):
        """Change a record's title and/or tags and index the new values."""
        current = self.get(record_id)
        updated = Record(
            record_id,
            current.title if title is None else title,
            current.tags if tags is None else tuple(tags),
        )
        self._records[record_id] = updated
        for term in derive_terms(updated):
            self.index.add(term, record_id)
        return updated

    def delete(self, record_id):
        """Remove a record and every index entry pointing at it."""
        record = self.get(record_id)
        for term in derive_terms(record):
            self.index.discard(term, record_id)
        del self._records[record_id]

path: src/catalog.py

content: """Catalog error types."""


class UnknownRecord(Exception):
    """No record exists with the requested id."""

path: src/errors.py

content: """The term index: posting lists from term to record ids."""

from .terms import derive_terms


class TermIndex:
    def __init__(self):
        self._postings = {}

    def add(self, term, record_id):
        self._postings.setdefault(term, set()).add(record_id)

    def discard(self, term, record_id):
        ids = self._postings.get(term)
        if ids:
            ids.discard(record_id)
            if not ids:
                del self._postings[term]

    def postings(self, term):
        """The ids indexed under term (a copy)."""
        return set(self._postings.get(term, ()))

    def rebuild(self, records):
        """Re-derive and re-index every record. Linear in catalog size —
        intended for bulk loads, not steady-state editing."""
        self._postings.clear()
        for record in records:
            for term in derive_terms(record):
                self.add(term, record.record_id)

    def term_count(self):
        return len(self._postings)

path: src/index.py

content: """Operational counters for capacity planning.

Production dashboards graph these per deployment; the derivations counter in
particular is watched closely, since term derivation is the catalog's main
CPU cost and is expected to stay flat per edit at steady state.
"""

derivations = 0
queries = 0


def count_derivation():
    global derivations
    derivations += 1


def count_query():
    global queries
    queries += 1


def reset():
    global derivations, queries
    derivations = 0
    queries = 0

path: src/metrics.py

content: """Catalog record model."""

from dataclasses import dataclass


@dataclass(frozen=True)
class Record:
    """A catalog entry. Tags are free-form labels; titles are searched by word."""

    record_id: str
    title: str
    tags: tuple = ()

path: src/records.py

content: """Plain-text rendering of catalog contents."""


def record_line(record):
    """One row: 'id  title  [tag, tag]'."""
    tags = ", ".join(record.tags)
    return "{}  {}  [{}]".format(record.record_id, record.title, tags)


def catalog_listing(catalog, record_ids):
    """Rows for the given ids, in the order given."""
    return [record_line(catalog.get(record_id)) for record_id in record_ids]

path: src/report.py

content: """Search over the catalog's term index."""

from . import metrics


def find(catalog, term):
    """Sorted ids of records indexed under term (case-insensitive)."""
    metrics.count_query()
    return sorted(catalog.index.postings(term.lower()))


def find_all(catalog, terms):
    """Ids indexed under every one of the given terms."""
    metrics.count_query()
    result = None
    for term in terms:
        ids = catalog.index.postings(term.lower())
        result = ids if result is None else result & ids
    return sorted(result or ())

path: src/search.py

content: """Term derivation: how a record is decomposed for the search index."""

from . import metrics


def derive_terms(record):
    """The searchable terms of a record: lowercased title words plus
    'tag:<label>' entries. Counted in metrics — derivation is the catalog's
    dominant per-record cost."""
    metrics.count_derivation()
    terms = {word for word in record.title.lower().split() if word}
    terms.update("tag:" + tag.lower() for tag in record.tags)
    return terms

path: src/terms.py

content: """Tests for the catalog, index, search, metrics, and reporting."""

import pytest

from src import metrics
from src.catalog import Catalog
from src.errors import UnknownRecord
from src.records import Record
from src.report import catalog_listing, record_line
from src.search import find, find_all


def test_add_and_find_case_insensitive():
    c = Catalog()
    c.add(Record("r1", "Quarterly Report", ("finance",)))
    assert find(c, "Quarterly") == ["r1"]
    assert find(c, "quarterly") == ["r1"]
    assert find(c, "tag:FINANCE") == ["r1"]
    assert find(c, "missing") == []


def test_find_sorted_and_find_all():
    c = Catalog()
    c.add(Record("r2", "fleet audit", ("ops",)))
    c.add(Record("r1", "fleet report", ("ops",)))
    assert find(c, "fleet") == ["r1", "r2"]
    assert find_all(c, ["fleet", "audit"]) == ["r2"]
    assert find_all(c, ["fleet", "tag:ops"]) == ["r1", "r2"]


def test_bulk_load_and_derivation_accounting():
    metrics.reset()
    c = Catalog()
    c.load([Record(f"r{i}", f"doc {i}", ("bulk",)) for i in range(25)])
    assert len(c) == 25
    assert metrics.derivations == 25
    assert find(c, "tag:bulk") == sorted(f"r{i}" for i in range(25))


def test_delete_cleans_search_and_unknown_raises():
    c = Catalog()
    c.add(Record("r1", "temp note", ("scratch",)))
    c.delete("r1")
    assert find(c, "temp") == []
    assert find(c, "tag:scratch") == []
    with pytest.raises(UnknownRecord):
        c.get("r1")
    with pytest.raises(UnknownRecord):
        c.delete("r1")


def test_update_indexes_new_values():
    c = Catalog()
    c.add(Record("r1", "draft plan", ("wip",)))
    updated = c.update("r1", title="final plan", tags=("approved",))
    assert updated.title == "final plan"
    assert find(c, "final") == ["r1"]
    assert find(c, "tag:approved") == ["r1"]
    assert c.get("r1").tags == ("approved",)


def test_query_counter():
    metrics.reset()
    c = Catalog()
    c.add(Record("r1", "solo", ()))
    find(c, "solo")
    find_all(c, ["solo"])
    assert metrics.queries == 2


def test_report_rendering():
    c = Catalog()
    c.add(Record("r1", "fleet audit", ("ops", "q3")))
    assert record_line(c.get("r1")) == "r1  fleet audit  [ops, q3]"
    assert catalog_listing(c, ["r1"]) == ["r1  fleet audit  [ops, q3]"]

path: tests/public/test_public.py

Reference patch:
--- a/src/catalog.py
+++ b/src/catalog.py
@@ -34,13 +34,17 @@ class Catalog:
     def update(self, record_id, title=None, tags=None):
         """Change a record's title and/or tags and index the new values."""
         current = self.get(record_id)
+        old_terms = derive_terms(current)
         updated = Record(
             record_id,
             current.title if title is None else title,
             current.tags if tags is None else tuple(tags),
         )
         self._records[record_id] = updated
-        for term in derive_terms(updated):
+        new_terms = derive_terms(updated)
+        for term in old_terms - new_terms:
+            self.index.discard(term, record_id)
+        for term in new_terms - old_terms:
             self.index.add(term, record_id)
         return updated
 


Hidden acceptance tests (diff):
diff --git a/tests/hidden/test_hidden.py b/tests/hidden/test_hidden.py
new file mode 100644
--- /dev/null
+++ b/tests/hidden/test_hidden.py
@@ -0,0 +1,101 @@
+"""Hidden evaluation tests. Never agent-visible (DEC-015).
+
+FAIL_TO_PASS: update() indexes a record's new terms but never removes the
+terms an edit dropped, so searches keep returning records for old values;
+every test below exercises staleness, so all seven fail at baseline. Graded
+gradient (DEC-036/DEC-042): the two bounded-work tests fuse correctness with
+derivation-count limits, so the correct-but-unbounded repairs land at 5/7 —
+rebuilding the index per edit fails the steady-state and churn bounds;
+verifying candidates at query time fails the query-work and churn bounds.
+Only the incremental fix (diff old terms against new in the update path)
+passes 7/7.
+"""
+
+from src import metrics
+from src.catalog import Catalog
+from src.records import Record
+from src.search import find
+
+
+def test_removed_tag_stops_matching():
+    c = Catalog()
+    c.add(Record("r1", "incident review", ("urgent", "ops")))
+    c.update("r1", tags=("ops",))
+    assert find(c, "tag:urgent") == []
+    assert find(c, "tag:ops") == ["r1"]
+
+
+def test_retitled_record_stops_matching_old_words():
+    c = Catalog()
+    c.add(Record("r1", "alpha report", ()))
+    c.update("r1", title="beta summary")
+    assert find(c, "alpha") == []
+    assert find(c, "report") == []
+    assert find(c, "beta") == ["r1"]
+    assert find(c, "summary") == ["r1"]
+
+
+def test_repeated_edits_leave_no_residue():
+    c = Catalog()
+    c.add(Record("r1", "one", ("a",)))
+    c.update("r1", title="two", tags=("b",))
+    c.update("r1", title="three", tags=("c",))
+    c.update("r1", title="four", tags=("d",))
+    for stale in ("one", "two", "three", "tag:a", "tag:b", "tag:c"):
+        assert find(c, stale) == [], stale
+    assert find(c, "four") == ["r1"]
+    assert find(c, "tag:d") == ["r1"]
+
+
+def test_overlapping_edit_keeps_shared_terms():
+    c = Catalog()
+    c.add(Record("r1", "fleet plan", ("keep", "drop")))
+    c.update("r1", tags=("keep", "new"))
+    assert find(c, "tag:keep") == ["r1"]
+    assert find(c, "tag:drop") == []
+    assert find(c, "tag:new") == ["r1"]
+    assert find(c, "fleet") == ["r1"]
+
+
+def test_steady_state_edit_work_stays_flat():
+    c = Catalog()
+    c.load([Record(f"r{i}", f"doc number {i}", ("bulk",)) for i in range(300)])
+    metrics.reset()
+    for i in range(20):
+        c.update(f"r{i}", title=f"revised item {i}", tags=("edited",))
+    assert find(c, "doc") == sorted(f"r{i}" for i in range(20, 300))
+    assert find(c, "revised") == sorted(f"r{i}" for i in range(20))
+    assert metrics.derivations <= 60  # a couple of derivations per edit
+
+
+def test_queries_do_no_derivation_work():
+    c = Catalog()
+    c.load([Record(f"r{i}", f"note {i}", ("archive",)) for i in range(300)])
+    c.update("r5", title="special memo", tags=("active",))
+    metrics.reset()
+    assert find(c, "special") == ["r5"]
+    assert find(c, "tag:active") == ["r5"]
+    for _ in range(8):
+        assert "r5" not in find(c, "tag:archive")
+    assert metrics.derivations == 0
+
+
+def test_churn_at_scale_stays_bounded():
+    c = Catalog()
+    c.load(
+        [Record(f"r{i:03d}", f"asset {i}", ("fleet",)) for i in range(100)]
+        + [Record(f"x{i:03d}", f"spare {i}", ("depot",)) for i in range(300)]
+    )
+    metrics.reset()
+    for i in range(30):
+        c.update(f"r{i:03d}", title=f"unit {i} serviced", tags=("fleet", "done"))
+    for i in range(10):
+        c.delete(f"x{i:03d}")
+    for _ in range(10):
+        hits = find(c, "tag:fleet")
+        assert len(hits) == 100
+    assert find(c, "asset") == sorted(f"r{i:03d}" for i in range(30, 100))
+    assert find(c, "unit") == sorted(f"r{i:03d}" for i in range(30))
+    assert find(c, "tag:done") == sorted(f"r{i:03d}" for i in range(30))
+    assert find(c, "spare") == sorted(f"x{i:03d}" for i in range(10, 300))
+    assert metrics.derivations <= 100