Domain: event-feed synchronization client (pagination cursors)

Issue:
Our nightly sync of the event feed keeps coming up short: every so often the pull finishes normally but ends up a few events light, and a full re-pull from scratch finds the stragglers. It tends to happen on busy days when bursts of events land within the same second. Nothing errors and nothing is duplicated — the sync just quietly misses some events. Expected behavior: a sync retrieves every event in the feed exactly once, no matter how timestamps cluster or how many pages it takes.

Repository files:
content: # Present so pytest adds the repository root to sys.path, making `import src`
# resolve from the test suite.

path: conftest.py

content: # Pure standard library unless the environment image pre-bakes wheels.

path: requirements.lock

path: src/__init__.py

content: """Sync client that drains the event feed page by page."""


def sync(feed, page_size=50):
    """Retrieve every event from the feed exactly once, oldest first."""
    events = []
    after_ts = None
    while True:
        batch = feed.fetch(after_ts=after_ts, limit=page_size)
        if not batch:
            return events
        events.extend(batch)
        after_ts = batch[-1].ts

path: src/client.py

content: """In-memory event feed with cursor-based pagination."""


class EventFeed:
    """Stores events and serves them in (ts, id) order."""

    def __init__(self, events=None):
        self._events = list(events or [])

    def add(self, event):
        """Publish an event to the feed."""
        self._events.append(event)

    def __len__(self):
        return len(self._events)

    def fetch(self, after_ts=None, after_id=None, limit=50):
        """Return up to limit events in (ts, id) order.

        With after_ts alone, only events whose ts is strictly greater are
        returned. Pass after_id — the id of the last event received — to
        resume within a timestamp: events sharing after_ts with a greater
        id are returned as well.
        """
        ordered = sorted(self._events, key=lambda event: (event.ts, event.id))
        if after_ts is not None:
            if after_id is not None:
                ordered = [
                    event
                    for event in ordered
                    if (event.ts, event.id) > (after_ts, after_id)
                ]
            else:
                ordered = [event for event in ordered if event.ts > after_ts]
        return ordered[:limit]

path: src/feed.py

content: """Data model for feed events."""

from dataclasses import dataclass


@dataclass(frozen=True)
class Event:
    """A single event published to the feed."""

    id: int
    ts: int
    kind: str
    data: str

path: src/models.py

content: """Aggregations over synced events."""

from collections import Counter


def count_by_kind(events):
    """Number of events of each kind."""
    return Counter(event.kind for event in events)


def latest(events):
    """The most recent event, or None for an empty list."""
    if not events:
        return None
    return max(events, key=lambda event: (event.ts, event.id))

path: src/stats.py

content: """Tests for the event feed, sync client, and stats."""

from src.client import sync
from src.feed import EventFeed
from src.models import Event
from src.stats import count_by_kind, latest


def make_feed(specs):
    feed = EventFeed()
    for event_id, ts, kind in specs:
        feed.add(Event(event_id, ts, kind, "payload-{}".format(event_id)))
    return feed


def test_fetch_orders_by_ts_then_id_and_respects_limit():
    feed = make_feed([(3, 20, "click"), (1, 10, "view"), (2, 20, "view")])
    assert [event.id for event in feed.fetch(limit=2)] == [1, 2]
    assert [event.id for event in feed.fetch()] == [1, 2, 3]


def test_fetch_after_ts_is_strictly_greater():
    feed = make_feed([(1, 10, "view"), (2, 20, "view"), (3, 30, "click")])
    assert [event.id for event in feed.fetch(after_ts=20)] == [3]


def test_fetch_after_id_resumes_within_a_timestamp():
    feed = make_feed(
        [(1, 10, "view"), (2, 10, "view"), (3, 10, "click"), (4, 20, "view")]
    )
    assert [event.id for event in feed.fetch(after_ts=10, after_id=2)] == [3, 4]


def test_sync_drains_multiple_pages():
    feed = make_feed([(i, i * 10, "view") for i in range(1, 8)])
    events = sync(feed, page_size=3)
    assert [event.id for event in events] == [1, 2, 3, 4, 5, 6, 7]


def test_sync_empty_feed():
    assert sync(EventFeed()) == []


def test_stats_counts_and_latest():
    events = sync(make_feed([(1, 10, "view"), (2, 20, "click"), (3, 30, "view")]))
    assert count_by_kind(events) == {"view": 2, "click": 1}
    assert latest(events).id == 3
    assert latest([]) is None

path: tests/public/test_public.py

Reference patch:
--- a/src/client.py
+++ b/src/client.py
@@ -5,9 +5,11 @@ def sync(feed, page_size=50):
     """Retrieve every event from the feed exactly once, oldest first."""
     events = []
     after_ts = None
+    after_id = None
     while True:
-        batch = feed.fetch(after_ts=after_ts, limit=page_size)
+        batch = feed.fetch(after_ts=after_ts, after_id=after_id, limit=page_size)
         if not batch:
             return events
         events.extend(batch)
         after_ts = batch[-1].ts
+        after_id = batch[-1].id


Hidden acceptance tests (diff):
diff --git a/tests/hidden/test_hidden.py b/tests/hidden/test_hidden.py
new file mode 100644
--- /dev/null
+++ b/tests/hidden/test_hidden.py
@@ -0,0 +1,48 @@
+"""Hidden evaluation tests. Never agent-visible (DEC-015).
+
+FAIL_TO_PASS: the client's resume cursor carries only the timestamp, so
+events sharing a timestamp that straddle a page boundary are dropped on the
+next fetch; all three tests fail on the buggy fixture. Once the client
+resumes with the id tiebreak the feed API provides, all three pass.
+"""
+
+from src.client import sync
+from src.feed import EventFeed
+from src.models import Event
+
+
+def make_feed(specs):
+    feed = EventFeed()
+    for event_id, ts, kind in specs:
+        feed.add(Event(event_id, ts, kind, "payload-{}".format(event_id)))
+    return feed
+
+
+def test_ties_straddling_a_page_boundary_are_delivered():
+    feed = make_feed(
+        [
+            (1, 10, "view"),
+            (2, 20, "view"),
+            (3, 20, "click"),
+            (4, 20, "view"),
+            (5, 30, "click"),
+            (6, 40, "view"),
+        ]
+    )
+    events = sync(feed, page_size=2)
+    assert [event.id for event in events] == [1, 2, 3, 4, 5, 6]
+
+
+def test_dense_ties_are_delivered_exactly_once():
+    feed = make_feed([(i, 7, "view") for i in range(1, 6)])
+    events = sync(feed, page_size=2)
+    assert sorted(event.id for event in events) == [1, 2, 3, 4, 5]
+    assert len(events) == 5
+
+
+def test_single_event_pages_survive_ties():
+    feed = make_feed(
+        [(1, 5, "view"), (2, 5, "click"), (3, 9, "view"), (4, 9, "click")]
+    )
+    events = sync(feed, page_size=1)
+    assert [event.id for event in events] == [1, 2, 3, 4]