Repository: rocicorp/mono
Pull request #4376: chore(zero-cache): logic for parsing a pg COPY stream
Author: login: darkgnotic
Created: 2025-05-14T19:37:09Z | Merged: 2025-05-14T20:02:15Z

This will be the basis for migrating initial-sync to use postgres COPY streams rather than cursor batching, allowing TCP backpressure to adapt the amount of buffered data as it arrives.

* TextTransform: implements the parsing of the [COPY text format](https://www.postgresql.org/docs/current/sql-copy.html) as it arrives in a NodeJS Readable stream. (Surprisingly, I couldn't find a suitable streaming TSV parser out there)
* RowTransform: applies the Postgres-client configured parsers to convert the text values to JS values, as it is done for non-streaming PG queries as well as for the replication stream.

This is logic and tests only. Integration into initial-sync is forthcoming.


Diff:
diff --git a/packages/zero-cache/src/db/pg-copy.pg-test.ts b/packages/zero-cache/src/db/pg-copy.pg-test.ts
new file mode 100644
index 0000000000..07e9af5779
Binary files /dev/null and b/packages/zero-cache/src/db/pg-copy.pg-test.ts differ
diff --git a/packages/zero-cache/src/db/pg-copy.ts b/packages/zero-cache/src/db/pg-copy.ts
new file mode 100644
index 0000000000..bf1807b99d
--- /dev/null
+++ b/packages/zero-cache/src/db/pg-copy.ts
@@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
+import {
+  assert,
+  assertArray,
+  assertNotUndefined,
+} from '../../../shared/src/asserts.ts';
+import type {PostgresDB} from '../types/pg.ts';
+import {getTypeParsers, type TypeParser} from './pg-type-parser.ts';
+
+/**
+ * A stream Transform that parses a Postgres `COPY ... TO` text stream
+ * into arrays of string / null values.
+ */
+export class TextTransform extends Transform {
+  #currRow: (string | null)[] | undefined;
+  #currCol: string | null | undefined;
+  #escaped = false;
+
+  constructor() {
+    super({objectMode: true});
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  _transform(
+    chunk: Buffer,
+    encoding: BufferEncoding,
+    callback: (e?: Error) => void,
+  ) {
+    try {
+      const text = chunk.toString(encoding);
+
+      let l = 0;
+      let r = 0;
+
+      const append = (text: string) => {
+        this.#currCol ??= '';
+        this.#currCol += text;
+      };
+
+      const flushSegmentIf = (condition: boolean) => {
+        if (condition) {
+          append(text.substring(l, r));
+        }
+        l = r + 1;
+      };
+
+      for (; r < text.length; r++) {
+        const ch = text[r];
+        if (this.#escaped) {
+          switch (ch) {
+            case 'N':
+              // \N is the NULL character.
+              if (this.#currCol !== undefined) {
+                throw new Error(
+                  `\\N escape sequence found in the middle of a column "${this.#currCol}"`,
+                );
+              }
+              this.#currCol = null;
+              break;
+            case '\\':
+              append('\\');
+              break;
+            case 'b':
+              append('\b');
+              break;
+            case 'f':
+              append('\f');
+              break;
+            case 'n':
+              append('\n');
+              break;
+            case 'r':
+              append('\r');
+              break;
+            case 't':
+              append('\t');
+              break;
+            case 'v':
+              append('\v');
+              break;
+            default:
+              // Not one of the escaped characters specified in https://www.postgresql.org/docs/current/sql-copy.html
+              throw new Error(`Unexpected escape character \\${ch}`);
+          }
+          l = r + 1;
+          this.#escaped = false;
+          continue;
+        }
+        switch (ch) {
+          case '\\':
+            flushSegmentIf(l < r);
+            this.#escaped = true;
+            break;
+
+          case '\t':
+          case '\n':
+            // Flush a possibly empty string unless #currCol was set to null via the \N escape.
+            flushSegmentIf(this.#currCol !== null);
+            assertNotUndefined(this.#currCol);
+
+            // Column is done in both cases.
+            (this.#currRow ??= []).push(this.#currCol);
+            this.#currCol = undefined;
+
+            if (ch === '\n') {
+              // Row is also done on \n
+              this.push(this.#currRow);
+              this.#currRow = undefined;
+            }
+            break;
+        }
+      }
+      flushSegmentIf(l < r);
+      callback();
+    } catch (e) {
+      callback(e instanceof Error ? e : new Error(String(e)));
+    }
+  }
+}
+
+/**
+ * A stream Transform that transforms the array of strings outputted by
+ * the {@link TextTransform} and parses them to their Javascript values
+ * as directed by the supplied list of column types.
+ */
+export class RowTransform extends Transform {
+  static async create(
+    db: PostgresDB,
+    columns: {typeOID: number; dataType?: string}[],
+  ) {
+    const typeParsers = await getTypeParsers(db);
+    const columnParsers = columns.map(c =>
+      typeParsers.getTypeParser(c.typeOID),
+    );
+    return new RowTransform(columnParsers);
+  }
+
+  readonly #columnParsers: TypeParser[];
+
+  private constructor(columnParsers: TypeParser[]) {
+    super({objectMode: true});
+    this.#columnParsers = columnParsers;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  _transform(row: string[], _encoding: string, callback: (e?: Error) => void) {
+    try {
+      // sanity check that this is preceded by a TextTransform in the pipeline
+      assertArray(row);
+      assert(
+        row.length === this.#columnParsers.length,
+        `Expected row to have ${this.#columnParsers.length} values but found ${row.length}`,
+      );
+
+      const parsed = row.map((val, pos) =>
+        val === null ? null : this.#columnParsers[pos](val),
+      );
+      this.push(parsed);
+      callback();
+    } catch (e) {
+      callback(e instanceof Error ? e : new Error(String(e)));
+    }
+  }
+}
diff --git a/packages/zero-cache/src/db/pg-type-parser.ts b/packages/zero-cache/src/db/pg-type-parser.ts
new file mode 100644
index 0000000000..232847a07d
--- /dev/null
+++ b/packages/zero-cache/src/db/pg-type-parser.ts
@@ -0,0 +1,55 @@
+import {assert} from '../../../shared/src/asserts.ts';
+import {mapValues} from '../../../shared/src/objects.ts';
+import type {PostgresDB} from '../types/pg.ts';
+
+// Arbitrary array type to test if the PostgresDB client has fetched types.
+const INT4_ARRAY_TYPE = 1007;
+
+export type TypeParser = (val: string) => unknown;
+export interface TypeParsers {
+  getTypeParser(typeOID: number): TypeParser;
+}
+
+// postgres.js has default type parsers with user-defined overrides
+// configurable per-client (see `postgresTypeConfig` in types/pg.ts).
+//
+// From these, the postgres.js client will automatically derive parsers
+// for array versions of these types, provided that the client was
+// configured with `fetch_types: true` (which is the default).
+//
+// A replication session (with `database: replication`), however, does
+// not support this type fetching, so it is done on a connection from
+// a default client.
+export async function getTypeParsers(db: PostgresDB): Promise<TypeParsers> {
+  if (!db.options.parsers[INT4_ARRAY_TYPE]) {
+    assert(db.options.fetch_types, `Supplied db must fetch_types`);
+
+    // Execute a query to ensure that fetchArrayTypes() gets executed:
+    // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/connection.js#L536
+    await db`SELECT 1`.simple();
+    assert(
+      db.options.parsers[INT4_ARRAY_TYPE],
+      `array types not fetched ${Object.keys(db.options.parsers)}`,
+    );
+  }
+  const parsers = mapValues(db.options.parsers, parse => {
+    // The postgres.js library tags parsers for array types with an `array: true` field.
+    // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/connection.js#L760
+    const isArrayType = (parse as unknown as {array?: boolean}).array;
+
+    // And then skips the first character when parsing the string,
+    // e.g. an array parser will parse '{1,2,3}' from '1,2,3}'.
+    // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/connection.js#L496
+    return isArrayType ? (val: string) => parse(val.substring(1)) : parse;
+  });
+  return {
+    // A type OID for which a parser is not explicitly defined
+    // is parsed as a String.
+    // https://github.com/porsager/postgres/blob/b0d8c8f363e006a74472d76f859da60c52a80368/src/connection.js#L494
+    //
+    // This is also consistent with the `pg` library, in which the absence of a
+    // TypeParser defaults to "noParse":
+    // https://github.com/brianc/node-pg-types/blob/5b26b826466cff4a9092b8c9e31960fe293ef3d9/index.js#L15
+    getTypeParser: typeOID => parsers[typeOID] ?? String,
+  };
+}
diff --git a/packages/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.ts b/packages/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.ts
index 2018251c28..f1644c0258 100644
--- a/packages/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.ts
+++ b/packages/zero-cache/src/services/change-source/pg/logical-replication/pgoutput-parser.ts
@@ -8,6 +8,7 @@
 
 export type TypeParser = (val: string) => unknown;
 
+import type {TypeParsers} from '../../../../db/pg-type-parser.ts';
 import {BinaryReader} from './binary-reader.ts';
 import type {
   Message,
@@ -29,15 +30,12 @@ export class PgoutputParser {
   _relationCache = new Map<number, MessageRelation>();
 
   // Replaces "pg-types" library.
-  #typeParsers: Record<number, TypeParser>;
-  constructor(typeParsers: Record<number, TypeParser>) {
+  #typeParsers: TypeParsers;
+  constructor(typeParsers: TypeParsers) {
     this.#typeParsers = typeParsers;
   }
   #getTypeParser(typeOid: number): TypeParser {
-    // The absence of a TypeParser defaults to "noParse", mimicking
-    // the behavior in pg-types:
-    // https://github.com/brianc/node-pg-types/blob/5b26b826466cff4a9092b8c9e31960fe293ef3d9/index.js#L15
-    return this.#typeParsers[typeOid] ?? String;
+    return this.#typeParsers.getTypeParser(typeOid);
   }
   // Replaced "pg-types" library.
 
diff --git a/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts b/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts
index 78247ece6d..2813c5f189 100644
--- a/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts
+++ b/packages/zero-cache/src/services/change-source/pg/logical-replication/stream.ts
@@ -5,9 +5,8 @@ import {
 import type {LogContext} from '@rocicorp/logger';
 import {defu} from 'defu';
 import postgres, {type Options, type PostgresType} from 'postgres';
-import {assert} from '../../../../../../shared/src/asserts.ts';
-import {mapValues} from '../../../../../../shared/src/objects.ts';
 import {sleep} from '../../../../../../shared/src/sleep.ts';
+import {getTypeParsers} from '../../../../db/pg-type-parser.ts';
 import {type PostgresDB} from '../../../../types/pg.ts';
 import {pipe, type Sink, type Source} from '../../../../types/streams.ts';
 import {Subscription} from '../../../../types/subscription.ts';
@@ -83,7 +82,7 @@ export async function subscribe(
   }, MANUAL_KEEPALIVE_TIMEOUT / 5);
 
   let destroyed = false;
-  const typeParsers = await getTypeParsers(lc, db);
+  const typeParsers = await getTypeParsers(db);
   const parser = new PgoutputParser(typeParsers);
   const messages = Subscription.create<StreamMessage>({
     cleanup: () => {
@@ -188,41 +187,3 @@ function makeAck(lsn: bigint): Buffer {
   x.writeBigInt64BE(microNow, 25);
   return x;
 }
-
-// Arbitrary array type to test if the PostgresDB client has fetched types.
-const INT4_ARRAY_TYPE = 1007;
-
-// postgres.js has default type parsers with user-defined overrides
-// configurable per-client (see `postgresTypeConfig` in types/pg.ts).
-//
-// From these, the postgres.js client will automatically derive parsers
-// for array versions of these types, provided that the client was
-// configured with `fetch_types: true` (which is the default).
-//
-// A replication session (with `database: replication`), however, does
-// not support this type fetching, so it is done on a connection from
-// a default client.
-async function getTypeParsers(lc: LogContext, db: PostgresDB) {
-  if (!db.options.parsers[INT4_ARRAY_TYPE]) {
-    assert(db.options.fetch_types, `Supplied db must fetch_types`);
-    lc.debug?.('fetching array types');
-
-    // Execute a query to ensure that fetchArrayTypes() gets executed:
-    // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/connection.js#L536
-    await db`SELECT 1`.simple();
-    assert(
-      db.options.parsers[INT4_ARRAY_TYPE],
-      `array types not fetched ${Object.keys(db.options.parsers)}`,
-    );
-  }
-  return mapValues(db.options.parsers, parse => {
-    // The postgres.js library tags parsers for array types with an `array: true` field.
-    // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/connection.js#L760
-    const isArrayType = (parse as unknown as {array?: boolean}).array;
-
-    // And then skips the first character when parsing the string,
-    // e.g. an array parser will parse '{1,2,3}' from '1,2,3}'.
-    // https://github.com/porsager/postgres/blob/089214e85c23c90cf142d47fb30bd03f42874984/src/connection.js#L496
-    return isArrayType ? (val: string) => parse(val.substring(1)) : parse;
-  });
-}


Reviews:
user: login: arv
body: LGTM
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T07:33:41Z
user: login: darkgnotic
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T15:17:30Z
user: login: darkgnotic
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T15:17:47Z
user: login: darkgnotic
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T15:18:00Z
user: login: darkgnotic
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T15:20:22Z

Review comments:
diff_hunk: @@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
path: packages/zero-cache/src/db/pg-copy.ts
user: login: arv
body: In general, is there a reason to use node:stream instead of web streams?
created_at: 2025-05-15T07:19:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
+import {
+  assert,
+  assertArray,
+  assertNotUndefined,
+} from '../../../shared/src/asserts.ts';
+import type {PostgresDB} from '../types/pg.ts';
+import {getTypeParsers, type TypeParser} from './pg-type-parser.ts';
+
+/**
+ * A stream Transform that parses a Postgres `COPY ... TO` text stream
+ * into arrays of string / null values.
+ */
+export class TextTransform extends Transform {
+  #currRow: (string | null)[] | undefined;
+  #currCol: string | null | undefined;
+  #escaped = false;
+
+  constructor() {
+    super({objectMode: true});
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  _transform(
+    chunk: Buffer,
+    encoding: BufferEncoding,
+    callback: (e?: Error) => void,
+  ) {
+    try {
+      const text = chunk.toString(encoding);
+
+      let l = 0;
+      let r = 0;
+
+      const append = (text: string) => {
+        this.#currCol ??= '';
+        this.#currCol += text;
+      };
+
+      const flushSegmentIf = (condition: boolean) => {
+        if (condition) {
+          append(text.substring(l, r));
+        }
+        l = r + 1;
+      };
+
+      for (; r < text.length; r++) {
+        const ch = text[r];
+        if (this.#escaped) {
+          switch (ch) {
+            case 'N':
path: packages/zero-cache/src/db/pg-copy.ts
user: login: arv
body: Blast from the past!

Use `charCodeAt` for these kind of things. It will use a SMIs which is stack allocated instead of a string which is heap allocated and give a nice perf increase.
created_at: 2025-05-15T07:21:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
+import {
+  assert,
+  assertArray,
+  assertNotUndefined,
+} from '../../../shared/src/asserts.ts';
+import type {PostgresDB} from '../types/pg.ts';
+import {getTypeParsers, type TypeParser} from './pg-type-parser.ts';
+
+/**
+ * A stream Transform that parses a Postgres `COPY ... TO` text stream
+ * into arrays of string / null values.
+ */
+export class TextTransform extends Transform {
+  #currRow: (string | null)[] | undefined;
+  #currCol: string | null | undefined;
+  #escaped = false;
+
+  constructor() {
+    super({objectMode: true});
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  _transform(
+    chunk: Buffer,
+    encoding: BufferEncoding,
+    callback: (e?: Error) => void,
+  ) {
+    try {
+      const text = chunk.toString(encoding);
+
+      let l = 0;
+      let r = 0;
+
+      const append = (text: string) => {
+        this.#currCol ??= '';
+        this.#currCol += text;
path: packages/zero-cache/src/db/pg-copy.ts
user: login: arv
body: micro optimize?
created_at: 2025-05-15T07:24:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
+import {
+  assert,
+  assertArray,
+  assertNotUndefined,
+} from '../../../shared/src/asserts.ts';
+import type {PostgresDB} from '../types/pg.ts';
+import {getTypeParsers, type TypeParser} from './pg-type-parser.ts';
+
+/**
+ * A stream Transform that parses a Postgres `COPY ... TO` text stream
+ * into arrays of string / null values.
+ */
+export class TextTransform extends Transform {
+  #currRow: (string | null)[] | undefined;
+  #currCol: string | null | undefined;
+  #escaped = false;
+
+  constructor() {
+    super({objectMode: true});
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  _transform(
+    chunk: Buffer,
+    encoding: BufferEncoding,
+    callback: (e?: Error) => void,
+  ) {
+    try {
+      const text = chunk.toString(encoding);
+
+      let l = 0;
+      let r = 0;
+
+      const append = (text: string) => {
+        this.#currCol ??= '';
+        this.#currCol += text;
+      };
+
+      const flushSegmentIf = (condition: boolean) => {
+        if (condition) {
+          append(text.substring(l, r));
+        }
+        l = r + 1;
+      };
+
+      for (; r < text.length; r++) {
+        const ch = text[r];
+        if (this.#escaped) {
+          switch (ch) {
+            case 'N':
+              // \N is the NULL character.
+              if (this.#currCol !== undefined) {
+                throw new Error(
+                  `\\N escape sequence found in the middle of a column "${this.#currCol}"`,
+                );
+              }
+              this.#currCol = null;
+              break;
+            case '\\':
+              append('\\');
+              break;
+            case 'b':
+              append('\b');
+              break;
+            case 'f':
+              append('\f');
+              break;
+            case 'n':
+              append('\n');
+              break;
+            case 'r':
+              append('\r');
+              break;
+            case 't':
+              append('\t');
+              break;
+            case 'v':
+              append('\v');
+              break;
+            default:
+              // Not one of the escaped characters specified in https://www.postgresql.org/docs/current/sql-copy.html
+              throw new Error(`Unexpected escape character \\${ch}`);
+          }
+          l = r + 1;
+          this.#escaped = false;
+          continue;
+        }
+        switch (ch) {
+          case '\\':
+            flushSegmentIf(l < r);
+            this.#escaped = true;
+            break;
+
+          case '\t':
+          case '\n':
+            // Flush a possibly empty string unless #currCol was set to null via the \N escape.
+            flushSegmentIf(this.#currCol !== null);
+            assertNotUndefined(this.#currCol);
+
+            // Column is done in both cases.
+            (this.#currRow ??= []).push(this.#currCol);
+            this.#currCol = undefined;
+
+            if (ch === '\n') {
+              // Row is also done on \n
+              this.push(this.#currRow);
+              this.#currRow = undefined;
+            }
+            break;
+        }
+      }
+      flushSegmentIf(l < r);
+      callback();
+    } catch (e) {
+      callback(e instanceof Error ? e : new Error(String(e)));
+    }
+  }
+}
+
+/**
+ * A stream Transform that transforms the array of strings outputted by
+ * the {@link TextTransform} and parses them to their Javascript values
+ * as directed by the supplied list of column types.
+ */
+export class RowTransform extends Transform {
+  static async create(
+    db: PostgresDB,
+    columns: {typeOID: number; dataType?: string}[],
+  ) {
+    const typeParsers = await getTypeParsers(db);
+    const columnParsers = columns.map(c =>
+      typeParsers.getTypeParser(c.typeOID),
+    );
+    return new RowTransform(columnParsers);
+  }
+
+  readonly #columnParsers: TypeParser[];
+
+  private constructor(columnParsers: TypeParser[]) {
path: packages/zero-cache/src/db/pg-copy.ts
user: login: arv
body: Why this dance with static create and private constructor?
created_at: 2025-05-15T07:26:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
+import {
+  assert,
+  assertArray,
+  assertNotUndefined,
+} from '../../../shared/src/asserts.ts';
+import type {PostgresDB} from '../types/pg.ts';
+import {getTypeParsers, type TypeParser} from './pg-type-parser.ts';
+
+/**
+ * A stream Transform that parses a Postgres `COPY ... TO` text stream
+ * into arrays of string / null values.
+ */
+export class TextTransform extends Transform {
+  #currRow: (string | null)[] | undefined;
+  #currCol: string | null | undefined;
+  #escaped = false;
+
+  constructor() {
+    super({objectMode: true});
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  _transform(
+    chunk: Buffer,
+    encoding: BufferEncoding,
+    callback: (e?: Error) => void,
+  ) {
+    try {
+      const text = chunk.toString(encoding);
+
+      let l = 0;
+      let r = 0;
+
+      const append = (text: string) => {
+        this.#currCol ??= '';
+        this.#currCol += text;
path: packages/zero-cache/src/db/pg-copy.ts
user: login: darkgnotic
body: > micro optimize?

Can you elaborate? I want to learn.  🙏 
created_at: 2025-05-15T15:17:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
+import {
+  assert,
+  assertArray,
+  assertNotUndefined,
+} from '../../../shared/src/asserts.ts';
+import type {PostgresDB} from '../types/pg.ts';
+import {getTypeParsers, type TypeParser} from './pg-type-parser.ts';
+
+/**
+ * A stream Transform that parses a Postgres `COPY ... TO` text stream
+ * into arrays of string / null values.
+ */
+export class TextTransform extends Transform {
+  #currRow: (string | null)[] | undefined;
+  #currCol: string | null | undefined;
+  #escaped = false;
+
+  constructor() {
+    super({objectMode: true});
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  _transform(
+    chunk: Buffer,
+    encoding: BufferEncoding,
+    callback: (e?: Error) => void,
+  ) {
+    try {
+      const text = chunk.toString(encoding);
+
+      let l = 0;
+      let r = 0;
+
+      const append = (text: string) => {
+        this.#currCol ??= '';
+        this.#currCol += text;
+      };
+
+      const flushSegmentIf = (condition: boolean) => {
+        if (condition) {
+          append(text.substring(l, r));
+        }
+        l = r + 1;
+      };
+
+      for (; r < text.length; r++) {
+        const ch = text[r];
+        if (this.#escaped) {
+          switch (ch) {
+            case 'N':
+              // \N is the NULL character.
+              if (this.#currCol !== undefined) {
+                throw new Error(
+                  `\\N escape sequence found in the middle of a column "${this.#currCol}"`,
+                );
+              }
+              this.#currCol = null;
+              break;
+            case '\\':
+              append('\\');
+              break;
+            case 'b':
+              append('\b');
+              break;
+            case 'f':
+              append('\f');
+              break;
+            case 'n':
+              append('\n');
+              break;
+            case 'r':
+              append('\r');
+              break;
+            case 't':
+              append('\t');
+              break;
+            case 'v':
+              append('\v');
+              break;
+            default:
+              // Not one of the escaped characters specified in https://www.postgresql.org/docs/current/sql-copy.html
+              throw new Error(`Unexpected escape character \\${ch}`);
+          }
+          l = r + 1;
+          this.#escaped = false;
+          continue;
+        }
+        switch (ch) {
+          case '\\':
+            flushSegmentIf(l < r);
+            this.#escaped = true;
+            break;
+
+          case '\t':
+          case '\n':
+            // Flush a possibly empty string unless #currCol was set to null via the \N escape.
+            flushSegmentIf(this.#currCol !== null);
+            assertNotUndefined(this.#currCol);
+
+            // Column is done in both cases.
+            (this.#currRow ??= []).push(this.#currCol);
+            this.#currCol = undefined;
+
+            if (ch === '\n') {
+              // Row is also done on \n
+              this.push(this.#currRow);
+              this.#currRow = undefined;
+            }
+            break;
+        }
+      }
+      flushSegmentIf(l < r);
+      callback();
+    } catch (e) {
+      callback(e instanceof Error ? e : new Error(String(e)));
+    }
+  }
+}
+
+/**
+ * A stream Transform that transforms the array of strings outputted by
+ * the {@link TextTransform} and parses them to their Javascript values
+ * as directed by the supplied list of column types.
+ */
+export class RowTransform extends Transform {
+  static async create(
+    db: PostgresDB,
+    columns: {typeOID: number; dataType?: string}[],
+  ) {
+    const typeParsers = await getTypeParsers(db);
+    const columnParsers = columns.map(c =>
+      typeParsers.getTypeParser(c.typeOID),
+    );
+    return new RowTransform(columnParsers);
+  }
+
+  readonly #columnParsers: TypeParser[];
+
+  private constructor(columnParsers: TypeParser[]) {
path: packages/zero-cache/src/db/pg-copy.ts
user: login: darkgnotic
body: The static create potentially blocks on I/O.
created_at: 2025-05-15T15:17:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
+import {
+  assert,
+  assertArray,
+  assertNotUndefined,
+} from '../../../shared/src/asserts.ts';
+import type {PostgresDB} from '../types/pg.ts';
+import {getTypeParsers, type TypeParser} from './pg-type-parser.ts';
+
+/**
+ * A stream Transform that parses a Postgres `COPY ... TO` text stream
+ * into arrays of string / null values.
+ */
+export class TextTransform extends Transform {
+  #currRow: (string | null)[] | undefined;
+  #currCol: string | null | undefined;
+  #escaped = false;
+
+  constructor() {
+    super({objectMode: true});
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  _transform(
+    chunk: Buffer,
+    encoding: BufferEncoding,
+    callback: (e?: Error) => void,
+  ) {
+    try {
+      const text = chunk.toString(encoding);
+
+      let l = 0;
+      let r = 0;
+
+      const append = (text: string) => {
+        this.#currCol ??= '';
+        this.#currCol += text;
+      };
+
+      const flushSegmentIf = (condition: boolean) => {
+        if (condition) {
+          append(text.substring(l, r));
+        }
+        l = r + 1;
+      };
+
+      for (; r < text.length; r++) {
+        const ch = text[r];
+        if (this.#escaped) {
+          switch (ch) {
+            case 'N':
path: packages/zero-cache/src/db/pg-copy.ts
user: login: darkgnotic
body: > Blast from the past!
> 
> Use `charCodeAt` for these kind of things. It will use a SMIs which is stack allocated instead of a string which is heap allocated and give a nice perf increase.

Got it. Thank you!
created_at: 2025-05-15T15:18:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,164 @@
+import {Transform} from 'node:stream';
path: packages/zero-cache/src/db/pg-copy.ts
user: login: darkgnotic
body: Oh, I don't know; I haven't looked at web streams. I would be interested in switching to the latter if it is better in some way.
created_at: 2025-05-15T15:20:22Z
author_association: CONTRIBUTOR

PR comments:
user: login: vercel[bot]
created_at: 2025-05-14T19:37:15Z
body: [vc]: #FHgdzA9LjG3PVuMdH7m7VyGEP4hsTgLiqJMWeouHu8E=:eyJpc01vbm9yZXBvIjp0cnVlLCJ0eXBlIjoiZ2l0aHViIiwicHJvamVjdHMiOlt7Im5hbWUiOiJyZXBsaWNhY2hlLWRvY3MiLCJyb290RGlyZWN0b3J5IjpudWxsLCJpbnNwZWN0b3JVcmwiOiJodHRwczovL3ZlcmNlbC5jb20vcm9jaWNvcnAvcmVwbGljYWNoZS1kb2NzLzhXY3NNRkFReExxRUVScDVTeHhKY05aeDZZMUoiLCJwcmV2aWV3VXJsIjoicmVwbGljYWNoZS1kb2NzLWdpdC1kYXJrZ25vdGljLXN0cmVhbWluZy1wZy1jb3B5LWxvZ2ljLXJvY2ljb3JwLnZlcmNlbC5hcHAiLCJuZXh0Q29tbWl0U3RhdHVzIjoiREVQTE9ZRUQiLCJsaXZlRmVlZGJhY2siOnsicmVzb2x2ZWQiOjAsInVucmVzb2x2ZWQiOjAsInRvdGFsIjowLCJsaW5rIjoicmVwbGljYWNoZS1kb2NzLWdpdC1kYXJrZ25vdGljLXN0cmVhbWluZy1wZy1jb3B5LWxvZ2ljLXJvY2ljb3JwLnZlcmNlbC5hcHAifX0seyJuYW1lIjoiemJ1Z3MiLCJyb290RGlyZWN0b3J5IjoiYXBwcy96YnVncyIsImluc3BlY3RvclVybCI6Imh0dHBzOi8vdmVyY2VsLmNvbS9yb2NpY29ycC96YnVncy8yMzI4YmlTaldIWWhaS1hZZmpKSEJnRHdoQXNuIiwicHJldmlld1VybCI6InpidWdzLWdpdC1kYXJrZ25vdGljLXN0cmVhbWluZy1wZy1jb3B5LWxvZ2ljLXJvY2ljb3JwLnZlcmNlbC5hcHAiLCJuZXh0Q29tbWl0U3RhdHVzIjoiREVQTE9ZRUQiLCJsaXZlRmVlZGJhY2siOnsicmVzb2x2ZWQiOjAsInVucmVzb2x2ZWQiOjAsInRvdGFsIjowLCJsaW5rIjoiemJ1Z3MtZ2l0LWRhcmtnbm90aWMtc3RyZWFtaW5nLXBnLWNvcHktbG9naWMtcm9jaWNvcnAudmVyY2VsLmFwcCJ9fV19
**The latest updates on your projects**. Learn more about [Vercel for Git ↗︎](https://vercel.link/github-learn-more)

| Name | Status | Preview | Comments | Updated (UTC) |
| :--- | :----- | :------ | :------- | :------ |
| **replicache-docs** | ✅ Ready ([Inspect](https://vercel.com/rocicorp/replicache-docs/8WcsMFAQxLqEERp5SxxJcNZx6Y1J)) | [Visit Preview](https://replicache-docs-git-darkgnotic-streaming-pg-copy-logic-rocicorp.vercel.app) | 💬 [**Add feedback**](https://vercel.live/open-feedback/replicache-docs-git-darkgnotic-streaming-pg-copy-logic-rocicorp.vercel.app?via=pr-comment-feedback-link) | May 14, 2025 8:00pm |
| **zbugs** | ✅ Ready ([Inspect](https://vercel.com/rocicorp/zbugs/2328biSjWHYhZKXYfjJHBgDwhAsn)) | [Visit Preview](https://zbugs-git-darkgnotic-streaming-pg-copy-logic-rocicorp.vercel.app) | 💬 [**Add feedback**](https://vercel.live/open-feedback/zbugs-git-darkgnotic-streaming-pg-copy-logic-rocicorp.vercel.app?via=pr-comment-feedback-link) | May 14, 2025 8:00pm |



author_association: NONE
user: login: darkgnotic
created_at: 2025-05-14T19:38:15Z
body: btw, the diff doesn't display the test because it contains binary characters, but it is a pretty good / thorough test

please let me know if you can think of any edge cases that we should add to the test.
author_association: CONTRIBUTOR
user: login: github-actions[bot]
created_at: 2025-05-14T19:39:14Z
body: <h2><a href="https://bencher.dev/perf/zero/reports/64eba1a0-b449-42f4-a488-da202c7ab10a?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero"><img src="https://bencher.dev/favicon.svg" width="24" height="24" alt="🐰" /> Bencher Report</a></h2><table><tr><td>Branch</td><td><a href="https://bencher.dev/perf/zero/branches/darkgnotic-streaming-pg-copy-logic?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">darkgnotic/streaming-pg-copy-logic</a></td></tr><tr><td>Testbed</td><td><a href="https://bencher.dev/perf/zero/testbeds/linux?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">Linux</a></td></tr></table><details><summary>Click to view all benchmark results</summary><br /><table><thead><tr><th>Benchmark</th><th><a href="https://bencher.dev/perf/zero/measures/file-size?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">File Size</a></th><th>Benchmark Result<br />kilobytes (KB)<br />(Result Δ%)</th><th>Upper Boundary<br />kilobytes (KB)<br />(Limit %)</th></tr></thead><tbody><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/zero-package-tgz?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">zero-package.tgz</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=f896796a-45a3-419e-bd66-cfc15c26b60e&heads=540069e7-7b42-4c62-b482-1d41ea38e9d7&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=2bcdd867-a170-440b-8c18-ee15b6476b90&measures=8d8dbe98-8c7a-4838-b351-cd9d621320fe&start_time=1744660868000&end_time=1747252868000&report=64eba1a0-b449-42f4-a488-da202c7ab10a&upper_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/037aee65-1b38-443a-badf-8195c4542c2e?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>1,166.11 KB<br /><details><summary>(+0.02%)</summary>Baseline: 1,165.88 KB</details></td><td>1,189.19 KB<br />(98.06%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/zero-js?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">zero.js</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=f896796a-45a3-419e-bd66-cfc15c26b60e&heads=540069e7-7b42-4c62-b482-1d41ea38e9d7&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=4f172dbf-1d31-4809-9c2d-12c79ed4c9d4&measures=8d8dbe98-8c7a-4838-b351-cd9d621320fe&start_time=1744660868000&end_time=1747252868000&report=64eba1a0-b449-42f4-a488-da202c7ab10a&upper_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/037aee65-1b38-443a-badf-8195c4542c2e?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>194.60 KB<br /><details><summary>(0.00%)</summary>Baseline: 194.60 KB</details></td><td>198.49 KB<br />(98.04%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/zero-js-br?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">zero.js.br</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=f896796a-45a3-419e-bd66-cfc15c26b60e&heads=540069e7-7b42-4c62-b482-1d41ea38e9d7&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=f9d12b7c-f48c-41fc-a662-5019d902c9f9&measures=8d8dbe98-8c7a-4838-b351-cd9d621320fe&start_time=1744660868000&end_time=1747252868000&report=64eba1a0-b449-42f4-a488-da202c7ab10a&upper_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/037aee65-1b38-443a-badf-8195c4542c2e?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>54.62 KB<br /><details><summary>(0.00%)</summary>Baseline: 54.62 KB</details></td><td>55.71 KB<br />(98.04%)</td></tr></tbody></table></details><a href="https://bencher.dev/perf/zero/reports/64eba1a0-b449-42f4-a488-da202c7ab10a?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">🐰 View full continuous benchmarking report in Bencher</a><div id="bencher.dev/projects/zero/id/size"></div>
author_association: NONE
user: login: github-actions[bot]
created_at: 2025-05-14T19:39:35Z
body: <h2><a href="https://bencher.dev/perf/zero/reports/aec8a5ca-3a78-4fa7-b932-9d721f930c2a?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero"><img src="https://bencher.dev/favicon.svg" width="24" height="24" alt="🐰" /> Bencher Report</a></h2><table><tr><td>Branch</td><td><a href="https://bencher.dev/perf/zero/branches/darkgnotic-streaming-pg-copy-logic?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">darkgnotic/streaming-pg-copy-logic</a></td></tr><tr><td>Testbed</td><td><a href="https://bencher.dev/perf/zero/testbeds/linux?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">Linux</a></td></tr></table><details><summary>Click to view all benchmark results</summary><br /><table><thead><tr><th>Benchmark</th><th><a href="https://bencher.dev/perf/zero/measures/throughput?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">Throughput</a></th><th>Benchmark Result<br />operations / second (ops/s)<br />(Result Δ%)</th><th>Lower Boundary<br />operations / second (ops/s)<br />(Limit %)</th></tr></thead><tbody><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/src-client-custom-bench-ts-big-schema?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">src/client/custom.bench.ts > big schema</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=f896796a-45a3-419e-bd66-cfc15c26b60e&heads=5107adea-5a5a-4926-94a9-ac4875ec4df1&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=da25d28a-0b6b-4baa-8550-fb143072856d&measures=fc802276-c2ed-462d-9fa5-0c1fcf2deb8e&start_time=1744660910000&end_time=1747252910000&report=aec8a5ca-3a78-4fa7-b932-9d721f930c2a&lower_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/d7788f21-4e90-43d0-937b-fe2512f15bbf?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>330,125.93 ops/s<br /><details><summary>(+49.61%)</summary>Baseline: 220,658.93 ops/s</details></td><td>8,201.59 ops/s<br />(2.48%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/src-client-zero-bench-ts-basics-all-1000-rows-x-10-colu-rypc9f4e?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">src/client/zero.bench.ts > basics > All 1000 rows x 10 columns (numbers)</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=f896796a-45a3-419e-bd66-cfc15c26b60e&heads=5107adea-5a5a-4926-94a9-ac4875ec4df1&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=8ff168e6-e598-4cc2-a9c7-e5d09c4aefec&measures=fc802276-c2ed-462d-9fa5-0c1fcf2deb8e&start_time=1744660910000&end_time=1747252910000&report=aec8a5ca-3a78-4fa7-b932-9d721f930c2a&lower_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/d7788f21-4e90-43d0-937b-fe2512f15bbf?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>1,589.05 ops/s<br /><details><summary>(+54.79%)</summary>Baseline: 1,026.60 ops/s</details></td><td>78.60 ops/s<br />(4.95%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/src-client-zero-bench-ts-pk-compare-pk-n?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">src/client/zero.bench.ts > pk compare > pk = N</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=f896796a-45a3-419e-bd66-cfc15c26b60e&heads=5107adea-5a5a-4926-94a9-ac4875ec4df1&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=ed1c4f7e-ef3f-4184-9f6d-61c76f3bd686&measures=fc802276-c2ed-462d-9fa5-0c1fcf2deb8e&start_time=1744660910000&end_time=1747252910000&report=aec8a5ca-3a78-4fa7-b932-9d721f930c2a&lower_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/d7788f21-4e90-43d0-937b-fe2512f15bbf?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>29,486.21 ops/s<br /><details><summary>(+23.08%)</summary>Baseline: 23,956.11 ops/s</details></td><td>9,304.48 ops/s<br />(31.56%)</td></tr><tr><td><a href="https://bencher.dev/perf/zero/benchmarks/src-client-zero-bench-ts-with-filter-lower-rows-500-x-1-d9xjs9n1?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">src/client/zero.bench.ts > with filter > Lower rows 500 x 10 columns (numbers)</a></td><td>📈 <a href="https://bencher.dev/perf/zero?branches=f896796a-45a3-419e-bd66-cfc15c26b60e&heads=5107adea-5a5a-4926-94a9-ac4875ec4df1&testbeds=7274290c-8365-4158-ba73-19904b7ef4ae&benchmarks=ce024352-2a47-400d-b489-b9951bf428ac&measures=fc802276-c2ed-462d-9fa5-0c1fcf2deb8e&start_time=1744660910000&end_time=1747252910000&report=aec8a5ca-3a78-4fa7-b932-9d721f930c2a&lower_boundary=true">view plot</a><br />🚷 <a href="https://bencher.dev/perf/zero/thresholds/d7788f21-4e90-43d0-937b-fe2512f15bbf?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">view threshold</a></td><td>2,418.00 ops/s<br /><details><summary>(+50.25%)</summary>Baseline: 1,609.36 ops/s</details></td><td>101.14 ops/s<br />(4.18%)</td></tr></tbody></table></details><a href="https://bencher.dev/perf/zero/reports/aec8a5ca-3a78-4fa7-b932-9d721f930c2a?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr%2Bcomments&utm_term=zero">🐰 View full continuous benchmarking report in Bencher</a><div id="bencher.dev/projects/zero/id/perf"></div>
author_association: NONE
user: login: arv
created_at: 2025-05-15T07:12:58Z
body: > * TextTransform: implements the parsing of the [COPY text format](https://www.postgresql.org/docs/current/sql-copy.html) as it arrives in a NodeJS Readable stream. (Surprisingly, I couldn't find a suitable streaming TSV parser out there)

https://www.npmjs.com/package/csv-parse seems reasonable
author_association: CONTRIBUTOR