Repository: rocicorp/mono
Pull request #4263: feat(zero-cache): continuous changelog cleanup
Author: login: darkgnotic
Created: 2025-04-25T00:24:06Z | Merged: 2025-04-25T00:33:54Z

### Feature

Continuous changelog cleanup based on `litestream` watermark monitoring in the `change-streamer`

### Implementation

Continuous changelog cleanup leverages a feature called "watermark tracking" in the rocicorp fork of litestream: https://github.com/rocicorp/litestream/pull/3

A `BackupMonitor` service periodically polls the prometheus metrics endpoint exported by litestream to track the last replicated value of the `"_zero.replicationState"."stateVersion"` cell, configured via the `watermark-table` and `watermark-column` litestream options, respectively.

When a new watermark value has been replicated, a cleanup is scheduled to happen at or after the `replication time` + `restore duration`, the latter added to ensure that a `zero-cache` restoring the replica is able to catch up from the change log even if the restore takes a long time.

The change log purging logic is the same as that which is performed when a subscriber reports the watermark that it's replica was restored at.

<img width="1786" alt="Screenshot 2025-04-24 at 18 02 48" src="https://github.com/user-attachments/assets/bb253777-1c17-45bb-9db0-d9eecafee047" />



Diff:
diff --git a/package-lock.json b/package-lock.json
index 5d381d8386..865617f586 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -16457,6 +16457,13 @@
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
       "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw=="
     },
+    "node_modules/json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
+      "dev": true,
+      "license": "ISC"
+    },
     "node_modules/json5": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
@@ -18413,6 +18420,39 @@
         "tslib": "^2.0.3"
       }
     },
+    "node_modules/nock": {
+      "version": "14.0.4",
+      "resolved": "https://registry.npmjs.org/nock/-/nock-14.0.4.tgz",
+      "integrity": "sha512-86fh+gIKH8H02+y0/HKAOZZXn6OwgzXvl6JYwfjvKkoKxUWz54wIIDU/+w24xzMvk/R8pNVXOrvTubyl+Ml6cg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@mswjs/interceptors": "^0.38.5",
+        "json-stringify-safe": "^5.0.1",
+        "propagate": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=18.20.0 <20 || >=20.12.1"
+      }
+    },
+    "node_modules/nock/node_modules/@mswjs/interceptors": {
+      "version": "0.38.6",
+      "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.38.6.tgz",
+      "integrity": "sha512-qFlpmObPqeUs4u3oFYv/OM/xyX+pNa5TRAjqjvMhbGYlyMhzSrE5UfncL2rUcEeVfD9Gebgff73hPwqcOwJQNA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@open-draft/deferred-promise": "^2.2.0",
+        "@open-draft/logger": "^0.3.0",
+        "@open-draft/until": "^2.0.0",
+        "is-node-process": "^1.2.0",
+        "outvariant": "^1.4.3",
+        "strict-event-emitter": "^0.5.1"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/node-abi": {
       "version": "3.71.0",
       "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz",
@@ -19031,6 +19071,15 @@
       "integrity": "sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ==",
       "license": "ISC"
     },
+    "node_modules/parse-prometheus-text-format": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/parse-prometheus-text-format/-/parse-prometheus-text-format-1.1.1.tgz",
+      "integrity": "sha512-dBlhYVACjRdSqLMFe4/Q1l/Gd3UmXm8ruvsTi7J6ul3ih45AkzkVpI5XHV4aZ37juGZW5+3dGU5lwk+QLM9XJA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "shallow-equal": "^1.2.0"
+      }
+    },
     "node_modules/parse5": {
       "version": "7.2.1",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz",
@@ -20997,6 +21046,16 @@
       "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==",
       "license": "MIT"
     },
+    "node_modules/propagate": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz",
+      "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/proper-lockfile": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz",
@@ -22900,6 +22959,12 @@
         "node": ">=8"
       }
     },
+    "node_modules/shallow-equal": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/shallow-equal/-/shallow-equal-1.2.1.tgz",
+      "integrity": "sha512-S4vJDjHHMBaiZuT9NPb616CSmLf618jawtv3sufLl6ivK8WocjAo58cXwbRV1cgqxH0Qbv+iUt6m05eqEa2IRA==",
+      "license": "MIT"
+    },
     "node_modules/shallowequal": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
@@ -27151,6 +27216,7 @@
         "json-custom-numbers": "^3.1.1",
         "kasi": "^1.1.0",
         "nanoid": "^5.1.2",
+        "parse-prometheus-text-format": "^1.1.1",
         "pg-format": "npm:pg-format-fix@^1.0.5",
         "postgres": "^3.4.4",
         "prettier": "^3.5.3",
@@ -27219,6 +27285,7 @@
         "jose": "^5.9.3",
         "json-custom-numbers": "^3.1.1",
         "nanoid": "^5.1.2",
+        "parse-prometheus-text-format": "^1.1.1",
         "pg-format": "npm:pg-format-fix@^1.0.5",
         "postgres": "^3.4.4",
         "tsx": "^4.19.1",
@@ -27238,6 +27305,7 @@
         "@types/ws": "^8.5.12",
         "@vitest/runner": "3.0.8",
         "concurrently": "^8.2.1",
+        "nock": "^14.0.4",
         "shared": "0.0.0",
         "typescript": "~5.8.2",
         "vitest": "3.0.8"
@@ -31937,6 +32005,7 @@
         "json-custom-numbers": "^3.1.1",
         "kasi": "^1.1.0",
         "nanoid": "^5.1.2",
+        "parse-prometheus-text-format": "^1.1.1",
         "pg-format": "npm:pg-format-fix@^1.0.5",
         "postgres": "^3.4.4",
         "prettier": "^3.5.3",
@@ -38127,6 +38196,12 @@
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
       "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw=="
     },
+    "json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
+      "dev": true
+    },
     "json5": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
@@ -39466,6 +39541,33 @@
         "tslib": "^2.0.3"
       }
     },
+    "nock": {
+      "version": "14.0.4",
+      "resolved": "https://registry.npmjs.org/nock/-/nock-14.0.4.tgz",
+      "integrity": "sha512-86fh+gIKH8H02+y0/HKAOZZXn6OwgzXvl6JYwfjvKkoKxUWz54wIIDU/+w24xzMvk/R8pNVXOrvTubyl+Ml6cg==",
+      "dev": true,
+      "requires": {
+        "@mswjs/interceptors": "^0.38.5",
+        "json-stringify-safe": "^5.0.1",
+        "propagate": "^2.0.0"
+      },
+      "dependencies": {
+        "@mswjs/interceptors": {
+          "version": "0.38.6",
+          "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.38.6.tgz",
+          "integrity": "sha512-qFlpmObPqeUs4u3oFYv/OM/xyX+pNa5TRAjqjvMhbGYlyMhzSrE5UfncL2rUcEeVfD9Gebgff73hPwqcOwJQNA==",
+          "dev": true,
+          "requires": {
+            "@open-draft/deferred-promise": "^2.2.0",
+            "@open-draft/logger": "^0.3.0",
+            "@open-draft/until": "^2.0.0",
+            "is-node-process": "^1.2.0",
+            "outvariant": "^1.4.3",
+            "strict-event-emitter": "^0.5.1"
+          }
+        }
+      }
+    },
     "node-abi": {
       "version": "3.71.0",
       "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz",
@@ -39870,6 +39972,14 @@
       "resolved": "https://registry.npmjs.org/parse-numeric-range/-/parse-numeric-range-1.3.0.tgz",
       "integrity": "sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ=="
     },
+    "parse-prometheus-text-format": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/parse-prometheus-text-format/-/parse-prometheus-text-format-1.1.1.tgz",
+      "integrity": "sha512-dBlhYVACjRdSqLMFe4/Q1l/Gd3UmXm8ruvsTi7J6ul3ih45AkzkVpI5XHV4aZ37juGZW5+3dGU5lwk+QLM9XJA==",
+      "requires": {
+        "shallow-equal": "^1.2.0"
+      }
+    },
     "parse5": {
       "version": "7.2.1",
       "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz",
@@ -41056,6 +41166,12 @@
         }
       }
     },
+    "propagate": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz",
+      "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==",
+      "dev": true
+    },
     "proper-lockfile": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz",
@@ -42486,6 +42602,11 @@
         "kind-of": "^6.0.2"
       }
     },
+    "shallow-equal": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/shallow-equal/-/shallow-equal-1.2.1.tgz",
+      "integrity": "sha512-S4vJDjHHMBaiZuT9NPb616CSmLf618jawtv3sufLl6ivK8WocjAo58cXwbRV1cgqxH0Qbv+iUt6m05eqEa2IRA=="
+    },
     "shallowequal": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
@@ -45211,6 +45332,8 @@
         "jose": "^5.9.3",
         "json-custom-numbers": "^3.1.1",
         "nanoid": "^5.1.2",
+        "nock": "^14.0.4",
+        "parse-prometheus-text-format": "^1.1.1",
         "pg-format": "npm:pg-format-fix@^1.0.5",
         "postgres": "^3.4.4",
         "shared": "0.0.0",
diff --git a/packages/zero-cache/package.json b/packages/zero-cache/package.json
index f74447233b..8a5e3db904 100644
--- a/packages/zero-cache/package.json
+++ b/packages/zero-cache/package.json
@@ -43,6 +43,7 @@
     "jose": "^5.9.3",
     "json-custom-numbers": "^3.1.1",
     "nanoid": "^5.1.2",
+    "parse-prometheus-text-format": "^1.1.1",
     "pg-format": "npm:pg-format-fix@^1.0.5",
     "postgres": "^3.4.4",
     "tsx": "^4.19.1",
@@ -62,6 +63,7 @@
     "@types/ws": "^8.5.12",
     "@vitest/runner": "3.0.8",
     "concurrently": "^8.2.1",
+    "nock": "^14.0.4",
     "shared": "0.0.0",
     "typescript": "~5.8.2",
     "vitest": "3.0.8"
diff --git a/packages/zero-cache/src/config/zero-config.test.ts b/packages/zero-cache/src/config/zero-config.test.ts
index 89ced5ffb4..716628cb64 100644
--- a/packages/zero-cache/src/config/zero-config.test.ts
+++ b/packages/zero-cache/src/config/zero-config.test.ts
@@ -248,6 +248,13 @@ test('zero-cache --help', () => {
                                                                 The location of the litestream backup, usually an s3:// URL.                                      
                                                                 If set, the litestream-executable must also be specified.                                         
                                                                                                                                                                   
+     --litestream-port number                                   optional                                                                                          
+       ZERO_LITESTREAM_PORT env                                                                                                                                   
+                                                                Port on which litestream exports metrics, used to determine the replication                       
+                                                                watermark up to which it is safe to purge change log records.                                     
+                                                                                                                                                                  
+                                                                If unspecified, defaults to --port + 2.                                                           
+                                                                                                                                                                  
      --litestream-checkpoint-threshold-mb number                default: 40                                                                                       
        ZERO_LITESTREAM_CHECKPOINT_THRESHOLD_MB env                                                                                                                
                                                                 The size of the WAL file at which to perform an SQlite checkpoint to apply                        
@@ -292,6 +299,16 @@ test('zero-cache --help', () => {
                                                                                                                                                                   
                                                                 This requires a custom build of litestream (version 0.3.13+z0.0.1+).                              
                                                                                                                                                                   
+     --litestream-restore-duration-ms-estimate number           optional                                                                                          
+       ZERO_LITESTREAM_RESTORE_DURATION_MS_ESTIMATE env                                                                                                           
+                                                                The estimated time required to restore the replica from backup. This duration                     
+                                                                is used to determine when it is safe to purge change long entries; the                            
+                                                                change-streamer waits for at least this duration before purging changes                           
+                                                                that have been successfully backed up.                                                            
+                                                                                                                                                                  
+                                                                This can generally be left unspecified, as the server will compute the estimate                   
+                                                                based on an actual restore, using the initial-sync time for bootstrapping.                        
+                                                                                                                                                                  
      --storage-db-tmp-dir string                                optional                                                                                          
        ZERO_STORAGE_DB_TMP_DIR env                                                                                                                                
                                                                 tmp directory for IVM operator storage. Leave unset to use os.tmpdir()                            
diff --git a/packages/zero-cache/src/config/zero-config.ts b/packages/zero-cache/src/config/zero-config.ts
index b07434b566..c7e18b0b04 100644
--- a/packages/zero-cache/src/config/zero-config.ts
+++ b/packages/zero-cache/src/config/zero-config.ts
@@ -382,6 +382,16 @@ export const zeroOptions = {
       ],
     },
 
+    port: {
+      type: v.number().optional(),
+      desc: [
+        `Port on which litestream exports metrics, used to determine the replication`,
+        `watermark up to which it is safe to purge change log records.`,
+        ``,
+        `If unspecified, defaults to {bold --port} + 2.`,
+      ],
+    },
+
     checkpointThresholdMB: {
       type: v.number().default(40),
       desc: [
@@ -443,6 +453,19 @@ export const zeroOptions = {
         `This requires a custom build of litestream (version 0.3.13+z0.0.1+).`,
       ],
     },
+
+    restoreDurationMsEstimate: {
+      type: v.number().optional(),
+      desc: [
+        `The estimated time required to restore the replica from backup. This duration`,
+        `is used to determine when it is safe to purge change long entries; the`,
+        `{bold change-streamer} waits for at least this duration before purging changes`,
+        `that have been successfully backed up.`,
+        ``,
+        `This can generally be left unspecified, as the server will compute the estimate`,
+        `based on an actual restore, using the initial-sync time for bootstrapping.`,
+      ],
+    },
   },
 
   storageDBTmpDir: {
diff --git a/packages/zero-cache/src/server/change-streamer.ts b/packages/zero-cache/src/server/change-streamer.ts
index c49e4391af..0a4bd31363 100644
--- a/packages/zero-cache/src/server/change-streamer.ts
+++ b/packages/zero-cache/src/server/change-streamer.ts
@@ -6,11 +6,13 @@ import {deleteLiteDB} from '../db/delete-lite-db.ts';
 import {warmupConnections} from '../db/warmup.ts';
 import {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';
 import {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';
+import {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';
 import {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';
 import {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';
 import type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';
 import {AutoResetSignal} from '../services/change-streamer/schema/tables.ts';
 import {exitAfter, runUntilKilled} from '../services/life-cycle.ts';
+import type {Service} from '../services/service.ts';
 import {pgClient} from '../types/pg.ts';
 import {
   parentWorker,
@@ -32,6 +34,7 @@ export default async function runWorker(
     change,
     replica,
     initialSync,
+    litestream,
   } = config;
   const lc = createLogContext(config, {worker: 'change-streamer'});
 
@@ -46,10 +49,12 @@ export default async function runWorker(
   const shard = getShardConfig(config);
 
   let changeStreamer: ChangeStreamerService | undefined;
+  let initialSyncTime: number | undefined;
 
   for (const first of [true, false]) {
     try {
       // Note: This performs initial sync of the replica if necessary.
+      const start = Date.now();
       const {changeSource, subscriptionState} =
         upstream.type === 'pg'
           ? await initializePostgresChangeSource(
@@ -65,6 +70,7 @@ export default async function runWorker(
               shard,
               replica.file,
             );
+      initialSyncTime = Date.now() - start;
 
       changeStreamer = await initializeStreamer(
         lc,
@@ -105,7 +111,20 @@ export default async function runWorker(
 
   parent.send(['ready', {ready: true}]);
 
-  return runUntilKilled(lc, parent, changeStreamer, changeStreamerWebServer);
+  const services: Service[] = [changeStreamer, changeStreamerWebServer];
+  if (litestream.backupURL) {
+    const {port: metricsPort = config.port + 2} = litestream;
+    services.push(
+      new BackupMonitor(
+        lc,
+        `http://localhost:${metricsPort}/metrics`,
+        changeStreamer,
+        litestream.restoreDurationMsEstimate ?? initialSyncTime,
+      ),
+    );
+  }
+
+  return runUntilKilled(lc, parent, ...services);
 }
 
 // fork()
diff --git a/packages/zero-cache/src/server/main.ts b/packages/zero-cache/src/server/main.ts
index 0e4f40eb6e..16e8464fca 100644
--- a/packages/zero-cache/src/server/main.ts
+++ b/packages/zero-cache/src/server/main.ts
@@ -96,7 +96,18 @@ export default async function runWorker(
     // a restore once, allowing the backup to be absent.
     // For view-syncers, attempt a restore for up to 10 times over 30 seconds.
     try {
-      await restoreReplica(lc, config, runChangeStreamer ? 1 : 10, 3000);
+      const restoreElapsedMs = await restoreReplica(
+        lc,
+        config,
+        runChangeStreamer ? 1 : 10,
+        3000,
+      );
+      if (!config.litestream.restoreDurationMsEstimate && restoreElapsedMs) {
+        internalFlags.push(
+          '--litestream-restore-duration-ms-estimate',
+          String(restoreElapsedMs),
+        );
+      }
     } catch (e) {
       if (runChangeStreamer) {
         // If the restore failed, e.g. due to a corrupt backup, the
diff --git a/packages/zero-cache/src/server/runner/config.test.ts b/packages/zero-cache/src/server/runner/config.test.ts
index 120712081b..f9af7ad6b3 100644
--- a/packages/zero-cache/src/server/runner/config.test.ts
+++ b/packages/zero-cache/src/server/runner/config.test.ts
@@ -501,6 +501,13 @@ test('zero-cache --help', () => {
                                                                 The location of the litestream backup, usually an s3:// URL.                                      
                                                                 If set, the litestream-executable must also be specified.                                         
                                                                                                                                                                   
+     --litestream-port number                                   optional                                                                                          
+       ZERO_LITESTREAM_PORT env                                                                                                                                   
+                                                                Port on which litestream exports metrics, used to determine the replication                       
+                                                                watermark up to which it is safe to purge change log records.                                     
+                                                                                                                                                                  
+                                                                If unspecified, defaults to --port + 2.                                                           
+                                                                                                                                                                  
      --litestream-checkpoint-threshold-mb number                default: 40                                                                                       
        ZERO_LITESTREAM_CHECKPOINT_THRESHOLD_MB env                                                                                                                
                                                                 The size of the WAL file at which to perform an SQlite checkpoint to apply                        
@@ -545,6 +552,16 @@ test('zero-cache --help', () => {
                                                                                                                                                                   
                                                                 This requires a custom build of litestream (version 0.3.13+z0.0.1+).                              
                                                                                                                                                                   
+     --litestream-restore-duration-ms-estimate number           optional                                                                                          
+       ZERO_LITESTREAM_RESTORE_DURATION_MS_ESTIMATE env                                                                                                           
+                                                                The estimated time required to restore the replica from backup. This duration                     
+                                                                is used to determine when it is safe to purge change long entries; the                            
+                                                                change-streamer waits for at least this duration before purging changes                           
+                                                                that have been successfully backed up.                                                            
+                                                                                                                                                                  
+                                                                This can generally be left unspecified, as the server will compute the estimate                   
+                                                                based on an actual restore, using the initial-sync time for bootstrapping.                        
+                                                                                                                                                                  
      --storage-db-tmp-dir string                                optional                                                                                          
        ZERO_STORAGE_DB_TMP_DIR env                                                                                                                                
                                                                 tmp directory for IVM operator storage. Leave unset to use os.tmpdir()                            
diff --git a/packages/zero-cache/src/server/runner/run-worker.ts b/packages/zero-cache/src/server/runner/run-worker.ts
index 2a334d8723..d73919b64f 100644
--- a/packages/zero-cache/src/server/runner/run-worker.ts
+++ b/packages/zero-cache/src/server/runner/run-worker.ts
@@ -43,7 +43,7 @@ export async function runWorker(
   // or change-streamer requests (i.e. by tenant `id`).
   const runAsReplicationManager = config.numSyncWorkers === 0;
 
-  // Start the first tenant at (port + 2) unless explicitly
+  // Start the first tenant at (port + 3) unless explicitly
   // overridden by its own ZERO_PORT ...
   let tenantPort = port;
   const tenants = config.tenants.map(tenant => {
@@ -51,7 +51,7 @@ export async function runWorker(
       ...process.env, // propagate all ENV variables from this process
       ...baseEnv, // defaults
       ['ZERO_TENANT_ID']: tenant.id,
-      ['ZERO_PORT']: String((tenantPort += 2)), // and bump the port by 2 thereafter.
+      ['ZERO_PORT']: String((tenantPort += 3)), // and bump the port by 3 thereafter.
       ...tenant.env, // overrides
     };
 
diff --git a/packages/zero-cache/src/services/change-streamer/backup-monitor.test.ts b/packages/zero-cache/src/services/change-streamer/backup-monitor.test.ts
new file mode 100644
index 0000000000..2394bc1c8c
--- /dev/null
+++ b/packages/zero-cache/src/services/change-streamer/backup-monitor.test.ts
@@ -0,0 +1,114 @@
+import nock from 'nock';
+import {beforeEach, describe, expect, test, vi} from 'vitest';
+import {createSilentLogContext} from '../../../../shared/src/logging-test-utils.ts';
+import {BackupMonitor} from './backup-monitor.ts';
+import type {ChangeStreamerService} from './change-streamer.ts';
+
+describe('change-streamer/backup-monitor', () => {
+  const scheduled: string[] = [];
+  const changeStreamer = {
+    scheduleCleanup: (watermark: string) => scheduled.push(watermark),
+  };
+  let metricsResponse = 'unconfigured';
+  let monitor: BackupMonitor;
+
+  function setMetricsResponse(watermark: string, timestamp: string) {
+    // Sample response from prometheus metrics handler
+    metricsResponse = `# HELP litestream_db_size The current size of the real DB
+# TYPE litestream_db_size gauge
+litestream_db_size{db="/tmp/zbugs-sync-replica.db"} 3.183935488e+09
+# HELP litestream_replica_progress The last replicated watermark and time of replication
+# TYPE litestream_replica_progress gauge
+litestream_replica_progress{db="/tmp/zbugs-sync-replica.db",name="file",watermark="${watermark}"} ${timestamp}
+# HELP litestream_replica_validation_total The number of validations performed
+# TYPE litestream_replica_validation_total counter
+litestream_replica_validation_total{db="/tmp/zbugs-sync-replica.db",name="file",status="error"} 0
+litestream_replica_validation_total{db="/tmp/zbugs-sync-replica.db",name="file",status="ok"} 0`;
+  }
+
+  beforeEach(() => {
+    const lc = createSilentLogContext();
+
+    vi.useFakeTimers();
+    scheduled.splice(0);
+
+    monitor = new BackupMonitor(
+      lc,
+      'http://localhost:4850/metrics',
+      changeStreamer as unknown as ChangeStreamerService,
+      100_000, // 100 seconds
+    );
+
+    nock('http://localhost:4850')
+      .persist()
+      .get('/metrics')
+      .reply(200, () => metricsResponse);
+
+    return () => vi.useRealTimers();
+  });
+
+  test('schedules overdue cleanup', async () => {
+    setMetricsResponse('618ocqq8', '1.74545644476593e+09');
+
+    await monitor.checkMetrics();
+
+    vi.advanceTimersByTime(0);
+    expect(scheduled).toEqual(['618ocqq8']);
+  });
+
+  test('schedules new cleanup', async () => {
+    vi.setSystemTime(Date.UTC(2025, 3, 24));
+    const nowSeconds = (Date.now() / 1000).toPrecision(9);
+    setMetricsResponse('618p0bw8', nowSeconds);
+
+    await monitor.checkMetrics();
+
+    vi.advanceTimersByTime(99_999);
+    expect(scheduled).toEqual([]);
+
+    vi.advanceTimersByTime(1);
+    expect(scheduled).toEqual(['618p0bw8']);
+  });
+
+  test('skips redundant watermarks cleanup', async () => {
+    vi.setSystemTime(Date.UTC(2025, 3, 24));
+    const firstWatermarkTime = (Date.UTC(2025, 3, 23) / 1000).toPrecision(9);
+    setMetricsResponse('618ofzts', firstWatermarkTime);
+
+    await monitor.checkMetrics();
+    vi.advanceTimersByTime(0);
+    expect(scheduled.pop()).toEqual('618ofzts');
+
+    for (let i = 0; i < 5; i++) {
+      await monitor.checkMetrics();
+      vi.advanceTimersByTime(100);
+      expect(scheduled).toEqual([]);
+    }
+
+    const secondWatermarkTime = ((Date.now() - 100_000) / 1000).toPrecision(9);
+    setMetricsResponse('618oko80', secondWatermarkTime);
+
+    await monitor.checkMetrics();
+    vi.advanceTimersByTime(1);
+    expect(scheduled.pop()).toEqual('618oko80');
+
+    for (let i = 0; i < 5; i++) {
+      await monitor.checkMetrics();
+      vi.advanceTimersByTime(100);
+      expect(scheduled).toEqual([]);
+    }
+
+    const thirdWatermarkTime = ((Date.now() - 100_000) / 1000).toPrecision(9);
+    setMetricsResponse('618p0bw8', thirdWatermarkTime);
+
+    await monitor.checkMetrics();
+    vi.advanceTimersByTime(1);
+    expect(scheduled.pop()).toEqual('618p0bw8');
+
+    for (let i = 0; i < 5; i++) {
+      await monitor.checkMetrics();
+      vi.advanceTimersByTime(100);
+      expect(scheduled).toEqual([]);
+    }
+  });
+});
diff --git a/packages/zero-cache/src/services/change-streamer/backup-monitor.ts b/packages/zero-cache/src/services/change-streamer/backup-monitor.ts
new file mode 100644
index 0000000000..2908dcc0a7
--- /dev/null
+++ b/packages/zero-cache/src/services/change-streamer/backup-monitor.ts
@@ -0,0 +1,132 @@
+import type {LogContext} from '@rocicorp/logger';
+import parsePrometheusTextFormat from 'parse-prometheus-text-format';
+import {promiseVoid} from '../../../../shared/src/resolved-promises.ts';
+import {RunningState} from '../running-state.ts';
+import type {Service} from '../service.ts';
+import type {ChangeStreamerService} from './change-streamer.ts';
+
+export const CHECK_INTERVAL_MS = 60 * 1000;
+const DEFAULT_CLEANUP_DELAY_MS = 60 * 1000;
+
+// The BackupMonitor polls the litestream "/metrics" endpoint to track the
+// watermark (label) value of the `litestream_replica_progress` gauge and
+// schedule cleanup of change log entries that can be purged as a result.
+//
+// See: https://github.com/rocicorp/litestream/pull/3
+//
+// Note that change log purging involves two time values:
+// 1. The time at which a watermark has been backed up by litestream
+// 2. The time it takes to restore the replica from the backup
+//
+// Namely, it is not safe to simply purge change log entries as soon as they
+// have been applied and backed up by litestream. Consider the case in which
+// litestream backs up new wal segments every minute, but it takes 5 minutes
+// to restore a replica: if a zero-cache starts restoring a replica at
+// minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
+// purging changelog records as soon as those watermarks are replicated would
+// result in the zero-cache not being able to catch up from minute 0 once it
+// has finished restoring the replica.
+//
+// Consequently, cleanup is delayed by an interval equivalent to the time it
+// takes for the litestream restore, which is calculated by the runner (when
+// it performed the restore or the initial sync). In addition to this delay,
+// the ChangeStreamerService itself adds at least 30 seconds of padding before
+// purging records, ensuring all active subscribers are past the watermark
+// before purging preceding records.
+export class BackupMonitor implements Service {
+  readonly id = 'backup-monitor';
+  readonly #lc: LogContext;
+  readonly #metricsEndpoint: string;
+  readonly #changeStreamer: ChangeStreamerService;
+  readonly #state = new RunningState(this.id);
+  readonly #cleanupDelayMs: number;
+  readonly #cleanupTimers = new Map<string, NodeJS.Timeout>();
+  #checkMetricsTimer: NodeJS.Timeout | undefined;
+
+  constructor(
+    lc: LogContext,
+    metricsEndpoint: string,
+    changeStreamer: ChangeStreamerService,
+    cleanupDelayMs: number | undefined,
+  ) {
+    this.#lc = lc.withContext('component', this.id);
+    this.#metricsEndpoint = metricsEndpoint;
+    this.#changeStreamer = changeStreamer;
+    this.#cleanupDelayMs = cleanupDelayMs ?? DEFAULT_CLEANUP_DELAY_MS;
+  }
+
+  run(): Promise<void> {
+    this.#lc.info?.(
+      `monitoring backups at ${this.#metricsEndpoint} with ` +
+        `${this.#cleanupDelayMs} ms cleanup delay`,
+    );
+    this.#checkMetricsTimer = setInterval(this.checkMetrics, CHECK_INTERVAL_MS);
+    return this.#state.stopped();
+  }
+
+  // Exported for testing
+  readonly checkMetrics = async () => {
+    try {
+      const resp = await fetch(this.#metricsEndpoint);
+      if (!resp.ok) {
+        this.#lc.warn?.(
+          `unable to fetch metrics at ${this.#metricsEndpoint}`,
+          await resp.text(),
+        );
+        return;
+      }
+      const families = parsePrometheusTextFormat(await resp.text());
+      for (const family of families) {
+        if (
+          family.type === 'GAUGE' &&
+          family.name === 'litestream_replica_progress'
+        ) {
+          for (const metric of family.metrics) {
+            const watermark = metric.labels?.watermark;
+            if (watermark && !this.#cleanupTimers.has(watermark)) {
+              const time = new Date(parseFloat(metric.value) * 1000);
+              const now = Date.now();
+              const cleanupAt = Math.max(
+                time.getTime() + this.#cleanupDelayMs,
+                now,
+              );
+              this.#lc.info?.(
+                `replicated watermark=${watermark} to ${metric.labels?.name}` +
+                  ` at ${time.toISOString()}. scheduling cleanup at ` +
+                  new Date(cleanupAt).toISOString() +
+                  ` (in ${cleanupAt - now} ms)`,
+              );
+              this.#cleanupTimers.set(
+                watermark,
+                setTimeout(() => {
+                  this.#changeStreamer.scheduleCleanup(watermark);
+                  // Clean up all previous watermarks, leaving the latest one in the
+                  // map to avoid redundant scheduling.
+                  for (const [key, timer] of this.#cleanupTimers.entries()) {
+                    if (key === watermark) {
+                      break;
+                    }
+                    this.#cleanupTimers.delete(key);
+                    clearTimeout(timer);
+                  }
+                }, cleanupAt - now),
+              );
+              this.#lc.debug?.('timers', [...this.#cleanupTimers.entries()]);
+            }
+          }
+        }
+      }
+    } catch (e) {
+      this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
+    }
+  };
+
+  stop(): Promise<void> {
+    clearInterval(this.#checkMetricsTimer);
+    for (const cleanupTimer of this.#cleanupTimers.values()) {
+      clearTimeout(cleanupTimer);
+    }
+    this.#state.stop(this.#lc);
+    return promiseVoid;
+  }
+}
diff --git a/packages/zero-cache/src/services/change-streamer/change-streamer-service.ts b/packages/zero-cache/src/services/change-streamer/change-streamer-service.ts
index 59d9a7fde9..c1aace1c74 100644
--- a/packages/zero-cache/src/services/change-streamer/change-streamer-service.ts
+++ b/packages/zero-cache/src/services/change-streamer/change-streamer-service.ts
@@ -431,13 +431,13 @@ class ChangeStreamerImpl implements ChangeStreamerService {
       this.#storer.catchup(subscriber, mode);
 
       if (initial) {
-        this.#scheduleCleanup(watermark);
+        this.scheduleCleanup(watermark);
       }
     }
     return Promise.resolve(downstream);
   }
 
-  #scheduleCleanup(watermark: string) {
+  scheduleCleanup(watermark: string) {
     const origSize = this.#initialWatermarks.size;
     this.#initialWatermarks.add(watermark);
 
diff --git a/packages/zero-cache/src/services/change-streamer/change-streamer.ts b/packages/zero-cache/src/services/change-streamer/change-streamer.ts
index 0860219551..c2c4d5d885 100644
--- a/packages/zero-cache/src/services/change-streamer/change-streamer.ts
+++ b/packages/zero-cache/src/services/change-streamer/change-streamer.ts
@@ -154,4 +154,11 @@ export type Error = v.Infer<typeof errorSchema>;
  */
 export type Downstream = v.Infer<typeof downstreamSchema>;
 
-export interface ChangeStreamerService extends ChangeStreamer, Service {}
+export interface ChangeStreamerService extends ChangeStreamer, Service {
+  /**
+   * Notifies the change streamer of a watermark that has been backed up,
+   * indicating that changes before the watermark can be purged if active
+   * subscribers have progressed beyond the watermark.
+   */
+  scheduleCleanup(watermark: string): void;
+}
diff --git a/packages/zero-cache/src/services/change-streamer/parse-prometheus-text-format.d.ts b/packages/zero-cache/src/services/change-streamer/parse-prometheus-text-format.d.ts
new file mode 100644
index 0000000000..afe790d089
--- /dev/null
+++ b/packages/zero-cache/src/services/change-streamer/parse-prometheus-text-format.d.ts
@@ -0,0 +1,54 @@
+// Types for https://github.com/yunyu/parse-prometheus-text-format base on:
+// https://github.com/yunyu/parse-prometheus-text-format/?tab=readme-ov-file#example
+declare module 'parse-prometheus-text-format' {
+  type Metric = {labels?: Record<string, string>};
+  type Counter = Metric & {value: string};
+  type Gauge = Metric & {value: string};
+  type Untyped = Metric & {value: string};
+  type Histogram = Metric & {
+    buckets: Record<string, number>;
+    count: string;
+    sum: string;
+  };
+  type Summary = Metric & {
+    quantiles?: Record<string, number>;
+    count: string;
+    sum: string;
+  };
+
+  type MetricFamilyDesc = {
+    name: string;
+    help: string;
+  };
+  type GaugeFamily = MetricFamilyDesc & {
+    type: 'GAUGE';
+    metrics: Gauge[];
+  };
+  type CounterFamily = MetricFamilyDesc & {
+    type: 'COUNTER';
+    metrics: Counter[];
+  };
+  type UntypedFamily = MetricFamilyDesc & {
+    type: 'UNTYPED';
+    metrics: Untyped[];
+  };
+  type HistogramFamily = MetricFamilyDesc & {
+    type: 'HISTOGRAM';
+    metrics: Histogram[];
+  };
+  type SummaryFamily = MetricFamilyDesc & {
+    type: 'SUMMARY';
+    metrics: Summary[];
+  };
+
+  type MetricFamily =
+    | GaugeFamily
+    | CounterFamily
+    | UntypedFamily
+    | HistogramFamily
+    | SummaryFamily;
+
+  export default function parsePrometheusTextFormat(
+    text: string,
+  ): MetricFamily[];
+}
diff --git a/packages/zero-cache/src/services/litestream/commands.ts b/packages/zero-cache/src/services/litestream/commands.ts
index 53b224bd51..7a800d7024 100644
--- a/packages/zero-cache/src/services/litestream/commands.ts
+++ b/packages/zero-cache/src/services/litestream/commands.ts
@@ -6,14 +6,21 @@ import {must} from '../../../../shared/src/must.ts';
 import {sleep} from '../../../../shared/src/sleep.ts';
 import type {ZeroConfig} from '../../config/zero-config.ts';
 
-type ZeroLitestreamConfig = Pick<ZeroConfig, 'log' | 'replica' | 'litestream'>;
+type ZeroLitestreamConfig = Pick<
+  ZeroConfig,
+  'port' | 'log' | 'replica' | 'litestream'
+>;
 
+/**
+ * @returns The time in milliseconds it took for the successful restore
+ *          (i.e. not counting failed attempts).
+ */
 export async function restoreReplica(
   lc: LogContext,
   config: ZeroLitestreamConfig,
   maxRetries: number,
   retryIntervalMs = 3000,
-) {
+): Promise<number | undefined> {
   for (let i = 0; i < maxRetries; i++) {
     if (i > 0) {
       lc.info?.(
@@ -21,13 +28,14 @@ export async function restoreReplica(
       );
       await sleep(retryIntervalMs);
     }
+    const start = Date.now();
     const restored = await tryRestore(config);
     if (restored) {
-      return;
+      return Date.now() - start;
     }
     if (maxRetries === 1) {
       lc.info?.('no litestream backup found');
-      return;
+      return undefined;
     }
   }
   throw new Error(`max attempts exceeded restoring replica`);
@@ -45,6 +53,7 @@ function getLitestream(
     backupURL,
     logLevel,
     configPath,
+    port = config.port + 2,
     checkpointThresholdMB,
     incrementalBackupIntervalMinutes,
     snapshotBackupIntervalHours,
@@ -77,6 +86,7 @@ function getLitestream(
       ),
       ['ZERO_LOG_FORMAT']: config.log.format,
       ['LITESTREAM_CONFIG']: configPath,
+      ['LITESTREAM_PORT']: String(port),
     },
   };
 }
diff --git a/packages/zero-cache/src/services/litestream/config.yml b/packages/zero-cache/src/services/litestream/config.yml
index c0c87f3f76..4a92088a1c 100644
--- a/packages/zero-cache/src/services/litestream/config.yml
+++ b/packages/zero-cache/src/services/litestream/config.yml
@@ -2,6 +2,8 @@ dbs:
   - path: ${ZERO_REPLICA_FILE}
     min-checkpoint-page-count: ${ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT}
     max-checkpoint-page-count: ${ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT}
+    watermark-table: _zero.replicationState
+    watermark-column: stateVersion
     replicas:
       - url: ${ZERO_LITESTREAM_BACKUP_URL}
         retention: ${ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES}m
@@ -11,3 +13,5 @@ dbs:
 logging:
   level: ${ZERO_LITESTREAM_LOG_LEVEL}
   type: ${ZERO_LOG_FORMAT}
+
+addr: localhost:${LITESTREAM_PORT}
diff --git a/packages/zero/Dockerfile b/packages/zero/Dockerfile
index 60656ed932..fdbaf1bdb7 100644
--- a/packages/zero/Dockerfile
+++ b/packages/zero/Dockerfile
@@ -4,8 +4,8 @@ WORKDIR /src/
 RUN git clone https://github.com/rocicorp/litestream.git
 WORKDIR /src/litestream/
 
-RUN git checkout 5c53233c50749a5f4e83915be77215dc1efce4c4
-ARG LITESTREAM_VERSION=0.3.13+z0.0.1  # upstream version + zero version
+RUN git checkout 5f52147c6ed5af87a3e4a0f952b02a82b2e2e0e2
+ARG LITESTREAM_VERSION=0.3.13+z0.0.2  # upstream version + zero version
 
 RUN --mount=type=cache,target=/root/.cache/go-build \
 	--mount=type=cache,target=/go/pkg \
diff --git a/packages/zero/package.json b/packages/zero/package.json
index a7eb46a356..e31a3dee4f 100644
--- a/packages/zero/package.json
+++ b/packages/zero/package.json
@@ -62,6 +62,7 @@
     "pg-format": "npm:pg-format-fix@^1.0.5",
     "postgres": "^3.4.4",
     "prettier": "^3.5.3",
+    "parse-prometheus-text-format": "^1.1.1",
     "semver": "^7.5.4",
     "tsx": "^4.19.1",
     "typedoc": "^0.28.2",
diff --git a/packages/zero/src/server/types.d.ts b/packages/zero/src/server/types.d.ts
new file mode 100644
index 0000000000..b891e8534b
--- /dev/null
+++ b/packages/zero/src/server/types.d.ts
@@ -0,0 +1 @@
+import '../../../zero-cache/src/services/change-streamer/parse-prometheus-text-format.d.ts';


Reviews:
user: login: tantaman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T13:50:17Z
user: login: tantaman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T13:52:51Z
user: login: tantaman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T13:56:36Z
user: login: darkgnotic
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T20:06:38Z
user: login: darkgnotic
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T20:07:53Z
user: login: darkgnotic
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-25T20:13:15Z

Review comments:
diff_hunk: @@ -46,10 +49,12 @@ export default async function runWorker(
   const shard = getShardConfig(config);
 
   let changeStreamer: ChangeStreamerService | undefined;
+  let initialSyncTime: number | undefined;
 
   for (const first of [true, false]) {
     try {
       // Note: This performs initial sync of the replica if necessary.
+      const start = Date.now();
path: packages/zero-cache/src/server/change-streamer.ts
user: login: tantaman
body: any reason to prefer `Date.now()` over `performance.now()`?
created_at: 2025-04-25T13:50:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -96,7 +96,18 @@ export default async function runWorker(
     // a restore once, allowing the backup to be absent.
     // For view-syncers, attempt a restore for up to 10 times over 30 seconds.
     try {
-      await restoreReplica(lc, config, runChangeStreamer ? 1 : 10, 3000);
+      const restoreElapsedMs = await restoreReplica(
+        lc,
+        config,
+        runChangeStreamer ? 1 : 10,
+        3000,
+      );
+      if (!config.litestream.restoreDurationMsEstimate && restoreElapsedMs) {
+        internalFlags.push(
+          '--litestream-restore-duration-ms-estimate',
+          String(restoreElapsedMs),
+        );
+      }
path: packages/zero-cache/src/server/main.ts
user: login: tantaman
body: Is it a problem if this value gets out of date? E.g., for a zero-cache that lives a very long time such that the replica size grows by a non-negligible amount.
created_at: 2025-04-25T13:52:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,132 @@
+import type {LogContext} from '@rocicorp/logger';
+import parsePrometheusTextFormat from 'parse-prometheus-text-format';
+import {promiseVoid} from '../../../../shared/src/resolved-promises.ts';
+import {RunningState} from '../running-state.ts';
+import type {Service} from '../service.ts';
+import type {ChangeStreamerService} from './change-streamer.ts';
+
+export const CHECK_INTERVAL_MS = 60 * 1000;
+const DEFAULT_CLEANUP_DELAY_MS = 60 * 1000;
+
+// The BackupMonitor polls the litestream "/metrics" endpoint to track the
+// watermark (label) value of the `litestream_replica_progress` gauge and
+// schedule cleanup of change log entries that can be purged as a result.
+//
+// See: https://github.com/rocicorp/litestream/pull/3
+//
+// Note that change log purging involves two time values:
+// 1. The time at which a watermark has been backed up by litestream
+// 2. The time it takes to restore the replica from the backup
+//
+// Namely, it is not safe to simply purge change log entries as soon as they
+// have been applied and backed up by litestream. Consider the case in which
+// litestream backs up new wal segments every minute, but it takes 5 minutes
+// to restore a replica: if a zero-cache starts restoring a replica at
+// minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
+// purging changelog records as soon as those watermarks are replicated would
+// result in the zero-cache not being able to catch up from minute 0 once it
+// has finished restoring the replica.
+//
+// Consequently, cleanup is delayed by an interval equivalent to the time it
+// takes for the litestream restore, which is calculated by the runner (when
+// it performed the restore or the initial sync). In addition to this delay,
+// the ChangeStreamerService itself adds at least 30 seconds of padding before
+// purging records, ensuring all active subscribers are past the watermark
+// before purging preceding records.
path: packages/zero-cache/src/services/change-streamer/backup-monitor.ts
user: login: tantaman
body: How hard would it be to track if there is an active restore happening and prevent purging until the restore completes? Maybe via a record in the upstream DB?

Is the timing of operations really consistent enough to not have to worry about this?
created_at: 2025-04-25T13:56:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -46,10 +49,12 @@ export default async function runWorker(
   const shard = getShardConfig(config);
 
   let changeStreamer: ChangeStreamerService | undefined;
+  let initialSyncTime: number | undefined;
 
   for (const first of [true, false]) {
     try {
       // Note: This performs initial sync of the replica if necessary.
+      const start = Date.now();
path: packages/zero-cache/src/server/change-streamer.ts
user: login: darkgnotic
body: Not really ... this is mostly out of habit. Is the latter better than the former (other than having higher precision)? Please educate me!   😄 
created_at: 2025-04-25T20:06:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -96,7 +96,18 @@ export default async function runWorker(
     // a restore once, allowing the backup to be absent.
     // For view-syncers, attempt a restore for up to 10 times over 30 seconds.
     try {
-      await restoreReplica(lc, config, runChangeStreamer ? 1 : 10, 3000);
+      const restoreElapsedMs = await restoreReplica(
+        lc,
+        config,
+        runChangeStreamer ? 1 : 10,
+        3000,
+      );
+      if (!config.litestream.restoreDurationMsEstimate && restoreElapsedMs) {
+        internalFlags.push(
+          '--litestream-restore-duration-ms-estimate',
+          String(restoreElapsedMs),
+        );
+      }
path: packages/zero-cache/src/server/main.ts
user: login: darkgnotic
body: This is a good point. We should think about this, and maybe factor in your next thought / question.
created_at: 2025-04-25T20:07:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,132 @@
+import type {LogContext} from '@rocicorp/logger';
+import parsePrometheusTextFormat from 'parse-prometheus-text-format';
+import {promiseVoid} from '../../../../shared/src/resolved-promises.ts';
+import {RunningState} from '../running-state.ts';
+import type {Service} from '../service.ts';
+import type {ChangeStreamerService} from './change-streamer.ts';
+
+export const CHECK_INTERVAL_MS = 60 * 1000;
+const DEFAULT_CLEANUP_DELAY_MS = 60 * 1000;
+
+// The BackupMonitor polls the litestream "/metrics" endpoint to track the
+// watermark (label) value of the `litestream_replica_progress` gauge and
+// schedule cleanup of change log entries that can be purged as a result.
+//
+// See: https://github.com/rocicorp/litestream/pull/3
+//
+// Note that change log purging involves two time values:
+// 1. The time at which a watermark has been backed up by litestream
+// 2. The time it takes to restore the replica from the backup
+//
+// Namely, it is not safe to simply purge change log entries as soon as they
+// have been applied and backed up by litestream. Consider the case in which
+// litestream backs up new wal segments every minute, but it takes 5 minutes
+// to restore a replica: if a zero-cache starts restoring a replica at
+// minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
+// purging changelog records as soon as those watermarks are replicated would
+// result in the zero-cache not being able to catch up from minute 0 once it
+// has finished restoring the replica.
+//
+// Consequently, cleanup is delayed by an interval equivalent to the time it
+// takes for the litestream restore, which is calculated by the runner (when
+// it performed the restore or the initial sync). In addition to this delay,
+// the ChangeStreamerService itself adds at least 30 seconds of padding before
+// purging records, ensuring all active subscribers are past the watermark
+// before purging preceding records.
path: packages/zero-cache/src/services/change-streamer/backup-monitor.ts
user: login: darkgnotic
body: I like this train of thought. Some ideas:
- Some row in the Change DB, which the change-streamer can check and/or lock on before purging records
- Would have to be careful to handle / distinguish orphaned rows
- Or ... maybe connect to the change-streamer via some different endpoint (like a pre-subscribe connection) to indicate that a subscription is coming soon and change-logs should not be purged until then.

I think the latter might be the cleanest. I will explore this more. I agree that the timing stuff is a bit "fuzzy" and would be nice to solidify. Thank you for the suggestion!
created_at: 2025-04-25T20:13:15Z
author_association: CONTRIBUTOR

PR comments:
user: login: vercel[bot]
created_at: 2025-04-25T00:24:10Z
body: [vc]: #PzCWNxBBnKXxzA3NTm0ZAEGL/dTAvCoO9krwky50xEI=:eyJpc01vbm9yZXBvIjp0cnVlLCJ0eXBlIjoiZ2l0aHViIiwicHJvamVjdHMiOlt7Im5hbWUiOiJyZXBsaWNhY2hlLWRvY3MiLCJyb290RGlyZWN0b3J5IjpudWxsLCJsaXZlRmVlZGJhY2siOnsicmVzb2x2ZWQiOjAsInVucmVzb2x2ZWQiOjAsInRvdGFsIjowLCJsaW5rIjoicmVwbGljYWNoZS1kb2NzLWdpdC1kYXJrZ25vdGljLWNvbnRpbnVvdXMtY2hhbmctMzZkMTQyLXJvY2ljb3JwLnZlcmNlbC5hcHAifSwiaW5zcGVjdG9yVXJsIjoiaHR0cHM6Ly92ZXJjZWwuY29tL3JvY2ljb3JwL3JlcGxpY2FjaGUtZG9jcy80dVJMdFh1aDlMcjFEVFhYVWVLbkszbVdmWDlFIiwicHJldmlld1VybCI6InJlcGxpY2FjaGUtZG9jcy1naXQtZGFya2dub3RpYy1jb250aW51b3VzLWNoYW5nLTM2ZDE0Mi1yb2NpY29ycC52ZXJjZWwuYXBwIiwibmV4dENvbW1pdFN0YXR1cyI6IkRFUExPWUVEIn0seyJuYW1lIjoiemJ1Z3MiLCJyb290RGlyZWN0b3J5IjoiYXBwcy96YnVncyIsImxpdmVGZWVkYmFjayI6eyJyZXNvbHZlZCI6MCwidW5yZXNvbHZlZCI6MCwidG90YWwiOjAsImxpbmsiOiIifSwiaW5zcGVjdG9yVXJsIjoiaHR0cHM6Ly92ZXJjZWwuY29tL3JvY2ljb3JwL3pidWdzLzlMcU56ZGtjcDhvSzd4b3JHZUw5MXdaTGpIcHAiLCJwcmV2aWV3VXJsIjoiemJ1Z3MtZ2l0LWRhcmtnbm90aWMtY29udGludW91cy1jaGFuZ2Vsb2ctY2xlYW51cC1yb2NpY29ycC52ZXJjZWwuYXBwIiwibmV4dENvbW1pdFN0YXR1cyI6IkRFUExPWUVEIn1dfQ==
**The latest updates on your projects**. Learn more about [Vercel for Git ↗︎](https://vercel.link/github-learn-more)

| Name | Status | Preview | Comments | Updated (UTC) |
| :--- | :----- | :------ | :------- | :------ |
| **replicache-docs** | ✅ Ready ([Inspect](https://vercel.com/rocicorp/replicache-docs/4uRLtXuh9Lr1DTXXUeKnK3mWfX9E)) | [Visit Preview](https://replicache-docs-git-darkgnotic-continuous-chang-36d142-rocicorp.vercel.app) | 💬 [**Add feedback**](https://vercel.live/open-feedback/replicache-docs-git-darkgnotic-continuous-chang-36d142-rocicorp.vercel.app?via=pr-comment-feedback-link) | Apr 25, 2025 0:29am |
| **zbugs** | ✅ Ready ([Inspect](https://vercel.com/rocicorp/zbugs/9LqNzdkcp8oK7xorGeL91wZLjHpp)) | [Visit Preview](https://zbugs-git-darkgnotic-continuous-changelog-cleanup-rocicorp.vercel.app) |  | Apr 25, 2025 0:29am |



author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-25T00:25:20Z
body: <h2><a href="https://bencher.dev/perf/zero/reports/e8429fbe-fbdc-4733-9431-83f6b3958111?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero"><img src="https://bencher.dev/favicon.svg" width="24" height="24" alt="🐰" /> Bencher Report</a></h2><table><tr><td>Branch</td><td><a href="https://bencher.dev/perf/zero/branches/darkgnotic-continuous-changelog-cleanup?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">darkgnotic/continuous-changelog-cleanup</a></td></tr><tr><td>Testbed</td><td><a href="https://bencher.dev/perf/zero/testbeds/linux?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">Linux</a></td></tr></table><details><summary>Click to view all benchmark results</summary><br /><table><thead><tr><th>Benchmark</th><th><a href="https://bencher.dev/perf/zero/measures/throughput?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">Throughput</a></th><th>Benchmark Result<br />operations / second (ops/s)<br />(Result Δ%)</th><th>Lower Boundary<br />operations / second (ops/s)<br />(Limit %)</th></tr></thead><tbody><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/src-client-custom-bench-ts-big-schema?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">src/client/custom.bench.ts > big schema</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=5d04e367-30ce-4ab8-94df-33e8ffa275f0&heads=72f393fb-681f-44d5-81fe-f648dd286749&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=da25d28a-0b6b-4baa-8550-fb143072856d&measures=fc802276-c2ed-462d-9fa5-0c1fcf2deb8e&start_time=1742951373000&end_time=1745543373000&report=e8429fbe-fbdc-4733-9431-83f6b3958111&lower_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/4cd67a3e-07ba-4991-ac80-36846c20e0ae?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>0.00 ops/s<br /><details><summary>(+117.56%)</summary>Baseline: 0.00 ops/s</details></td><td>0.00 ops/s<br />(13.02%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/src-client-zero-bench-ts-basics-all-1000-rows-x-10-colu-rypc9f4e?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">src/client/zero.bench.ts > basics > All 1000 rows x 10 columns (numbers)</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=5d04e367-30ce-4ab8-94df-33e8ffa275f0&heads=72f393fb-681f-44d5-81fe-f648dd286749&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=8ff168e6-e598-4cc2-a9c7-e5d09c4aefec&measures=fc802276-c2ed-462d-9fa5-0c1fcf2deb8e&start_time=1742951373000&end_time=1745543373000&report=e8429fbe-fbdc-4733-9431-83f6b3958111&lower_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/4cd67a3e-07ba-4991-ac80-36846c20e0ae?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>42.96 ops/s<br /><details><summary>(+7.41%)</summary>Baseline: 39.99 ops/s</details></td><td>37.28 ops/s<br />(86.77%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/src-client-zero-bench-ts-with-filter-lower-rows-500-x-1-d9xjs9n1?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">src/client/zero.bench.ts > with filter > Lower rows 500 x 10 columns (numbers)</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=5d04e367-30ce-4ab8-94df-33e8ffa275f0&heads=72f393fb-681f-44d5-81fe-f648dd286749&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=ce024352-2a47-400d-b489-b9951bf428ac&measures=fc802276-c2ed-462d-9fa5-0c1fcf2deb8e&start_time=1742951373000&end_time=1745543373000&report=e8429fbe-fbdc-4733-9431-83f6b3958111&lower_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/4cd67a3e-07ba-4991-ac80-36846c20e0ae?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>42.58 ops/s<br /><details><summary>(+6.89%)</summary>Baseline: 39.84 ops/s</details></td><td>36.06 ops/s<br />(84.69%)</td></tr></tbody></table></details><a href="https://bencher.dev/perf/zero/reports/e8429fbe-fbdc-4733-9431-83f6b3958111?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">🐰 View full continuous benchmarking report in Bencher</a><div id="bencher.dev/projects/zero/id/perf"></div>
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-25T00:30:31Z
body: <h2><a href="https://bencher.dev/perf/zero/reports/6fa2b62d-2087-4d5d-b53c-03a111574b55?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero"><img src="https://bencher.dev/favicon.svg" width="24" height="24" alt="🐰" /> Bencher Report</a></h2><table><tr><td>Branch</td><td><a href="https://bencher.dev/perf/zero/branches/darkgnotic-continuous-changelog-cleanup?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">darkgnotic/continuous-changelog-cleanup</a></td></tr><tr><td>Testbed</td><td><a href="https://bencher.dev/perf/zero/testbeds/linux?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">Linux</a></td></tr></table><details><summary>Click to view all benchmark results</summary><br /><table><thead><tr><th>Benchmark</th><th><a href="https://bencher.dev/perf/zero/measures/file-size?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">File Size</a></th><th>Benchmark Result<br />kilobytes (KB)<br />(Result Δ%)</th><th>Upper Boundary<br />kilobytes (KB)<br />(Limit %)</th></tr></thead><tbody><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/zero-package-tgz?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">zero-package.tgz</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=5d04e367-30ce-4ab8-94df-33e8ffa275f0&heads=3e34c39b-1f4e-4f6b-bab5-496a5469ceb2&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=2bcdd867-a170-440b-8c18-ee15b6476b90&measures=8d8dbe98-8c7a-4838-b351-cd9d621320fe&start_time=1742951067000&end_time=1745543067000&report=6fa2b62d-2087-4d5d-b53c-03a111574b55&upper_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/29332aaf-6d0d-4f58-93a0-bea3b6e9bc14?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>1,147.21 KB<br /><details><summary>(+0.16%)</summary>Baseline: 1,145.44 KB</details></td><td>1,168.34 KB<br />(98.19%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/zero-js?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">zero.js</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=5d04e367-30ce-4ab8-94df-33e8ffa275f0&heads=3e34c39b-1f4e-4f6b-bab5-496a5469ceb2&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=4f172dbf-1d31-4809-9c2d-12c79ed4c9d4&measures=8d8dbe98-8c7a-4838-b351-cd9d621320fe&start_time=1742951067000&end_time=1745543067000&report=6fa2b62d-2087-4d5d-b53c-03a111574b55&upper_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/29332aaf-6d0d-4f58-93a0-bea3b6e9bc14?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>194.61 KB<br /><details><summary>(0.00%)</summary>Baseline: 194.61 KB</details></td><td>198.51 KB<br />(98.04%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/zero-js-br?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">zero.js.br</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=5d04e367-30ce-4ab8-94df-33e8ffa275f0&heads=3e34c39b-1f4e-4f6b-bab5-496a5469ceb2&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=f9d12b7c-f48c-41fc-a662-5019d902c9f9&measures=8d8dbe98-8c7a-4838-b351-cd9d621320fe&start_time=1742951067000&end_time=1745543067000&report=6fa2b62d-2087-4d5d-b53c-03a111574b55&upper_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/29332aaf-6d0d-4f58-93a0-bea3b6e9bc14?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>54.56 KB<br /><details><summary>(0.00%)</summary>Baseline: 54.56 KB</details></td><td>55.65 KB<br />(98.04%)</td></tr></tbody></table></details><a href="https://bencher.dev/perf/zero/reports/6fa2b62d-2087-4d5d-b53c-03a111574b55?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">🐰 View full continuous benchmarking report in Bencher</a><div id="bencher.dev/projects/zero/id/size"></div>
author_association: NONE