Repository: ponder-sh/ponder (ts)

Issue:
bug: null characters throw error when inserted into `text` columns
### Related

https://github.com/ponder-sh/ponder/issues/1376

### Version

0.8.8, 0.7.17

### Current behavior

The server crashes the error (or a similar error for ASCII encoding):
```
FlushError: invalid byte sequence for encoding "UTF8": 0x00
```

### Expected behavior

Flush succeeds regardless of null characters

### Steps to reproduce

Don't have a repro handy, but in general this occurs because EVM `string` values can contain null characters, but the Postgres `text` data type does not allow them. We've seen null characters in decoded event `data` (very likely the source of the user reports we're seeing) as well as raw RPC fields like `trace.revertReason`.

### Link to repository

_No response_

### Anything else?

Two possible solutions:
1. Ensure that any values within the `event` object, particularly decoded event arguments (e.g. `event.args.{xxx}`), do not contain null characters by sanitizing them with `str.replace(/\0/g, '')`.
2. Sanitize any value being inserted into a `text` column at the driver level.

I prefer 1, at least as a starting point. On the raw RPC side of things, we should probably sanitize specific free text fields like revertReason before insertion.

Pull request description:
fix: null bytes
It's possible for decoded values for `string` ABI parameters to contain null characters. Null characters are ignored by almost all software systems, except the postgres `TEXT` data type does not accept them ([context](https://www.commandprompt.com/blog/null-characters-workarounds-arent-good-enough)).

This PR removes null characters from all decoded ABI strings.

Fixes #1378 #1376

Patch:
diff --git a/.changeset/eleven-onions-itch.md b/.changeset/eleven-onions-itch.md
new file mode 100644
index 00000000..97cdb1aa
--- /dev/null
+++ b/.changeset/eleven-onions-itch.md
@@ -0,0 +1,5 @@
+---
+"ponder": patch
+---
+
+Fixed the `FlushError: invalid byte sequence for encoding "UTF8": 0x00` error by removing null characters from decoded ABI parameters.
diff --git a/packages/core/src/sync/events.ts b/packages/core/src/sync/events.ts
index 6335a486..67a7a726 100644
--- a/packages/core/src/sync/events.ts
+++ b/packages/core/src/sync/events.ts
@@ -500,7 +500,7 @@ export const decodeEvents = (
 
                 event: {
                   name: safeName,
-                  args,
+                  args: removeNullCharacters(args),
                   log: event.log!,
                   block: event.block,
                   transaction: event.transaction!,
@@ -556,8 +556,8 @@ export const decodeEvents = (
                 name: `${source.name}.${safeName}`,
 
                 event: {
-                  args,
-                  result,
+                  args: removeNullCharacters(args),
+                  result: removeNullCharacters(result),
                   trace: event.trace!,
                   block: event.block,
                   transaction: event.transaction!,
@@ -661,7 +661,7 @@ export const decodeEvents = (
 };
 
 /** @see https://github.com/wevm/viem/blob/main/src/utils/abi/decodeEventLog.ts#L99 */
-function decodeEventLog({
+export function decodeEventLog({
   abiItem,
   topics,
   data,
@@ -731,6 +731,26 @@ function decodeTopic({ param, value }: { param: AbiParameter; value: Hex }) {
   return decodedArg[0];
 }
 
+export function removeNullCharacters(obj: unknown): unknown {
+  if (typeof obj === "string") {
+    return obj.replace(/\0/g, "");
+  }
+  if (Array.isArray(obj)) {
+    // Recursively handle array elements
+    return obj.map(removeNullCharacters);
+  }
+  if (obj && typeof obj === "object") {
+    // Recursively handle object properties
+    const newObj: { [key: string]: unknown } = {};
+    for (const [key, val] of Object.entries(obj)) {
+      newObj[key] = removeNullCharacters(val);
+    }
+    return newObj;
+  }
+  // For other types (number, boolean, null, undefined, etc.), return as-is
+  return obj;
+}
+
 const convertBlock = (block: SyncBlock): Block => ({
   baseFeePerGas: block.baseFeePerGas ? hexToBigInt(block.baseFeePerGas) : null,
   difficulty: hexToBigInt(block.difficulty),


Test patch:
diff --git a/packages/core/src/sync/events.test.ts b/packages/core/src/sync/events.test.ts
index 0d1f116f..839c5b9e 100644
--- a/packages/core/src/sync/events.test.ts
+++ b/packages/core/src/sync/events.test.ts
@@ -51,7 +51,9 @@ import {
   type TraceEvent,
   type TransferEvent,
   buildEvents,
+  decodeEventLog,
   decodeEvents,
+  removeNullCharacters,
 } from "./events.js";
 import type { LogFactory, LogFilter } from "./source.js";
 
@@ -832,3 +834,66 @@ test("buildEvents() matches getEvents() trace", async (context) => {
 
   await cleanup();
 });
+
+test("removeNullCharacters removes null characters", () => {
+  // NameRegistered event from this transaction contains null characters:
+  // https://etherscan.io/tx/0x2e67be22d5e700e61e102b926f28ba451c53a6cd6438c53b43dbb783c2081a12#eventlog
+  const log = {
+    topics: [
+      "0xca6abbe9d7f11422cb6ca7629fbf6fe9efb1c621f71ce8f02b9f2a230097404f",
+      "0x56e1003dc29ff83445ba93c493f4a76570eb667494e78c6974a745593131ae2a",
+      "0x0000000000000000000000008504a09352555ff1acf9c8a8d9fb5fdcc4161cbc",
+    ],
+    data: "0x0000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000697a5dd2a81dc000000000000000000000000000000000000000000000000000000006457430e000000000000000000000000000000000000000000000000000000000000001174656e63656e74636c7562000000000000000000000000000000000000000000",
+  } as const;
+
+  const abiItem = {
+    anonymous: false,
+    inputs: [
+      {
+        indexed: false,
+        internalType: "string",
+        name: "name",
+        type: "string",
+      },
+      {
+        indexed: true,
+        internalType: "bytes32",
+        name: "label",
+        type: "bytes32",
+      },
+      {
+        indexed: true,
+        internalType: "address",
+        name: "owner",
+        type: "address",
+      },
+      {
+        indexed: false,
+        internalType: "uint256",
+        name: "cost",
+        type: "uint256",
+      },
+      {
+        indexed: false,
+        internalType: "uint256",
+        name: "expires",
+        type: "uint256",
+      },
+    ],
+    name: "NameRegistered",
+    type: "event",
+  } as const;
+
+  const args = decodeEventLog({
+    abiItem,
+    topics: log.topics as unknown as [signature: Hex, ...args: Hex[]],
+    data: log.data,
+  });
+
+  expect(args.name).toBe("tencentclub\x00\x00\x00\x00\x00\x00");
+
+  const cleanedArgs = removeNullCharacters(args);
+
+  expect((cleanedArgs as any).name).toBe("tencentclub");
+});