Repository: unjs/h3
Pull request #847 (2024-07-23 16:31:36)

fix(iterable): always encode chunks
This PR fixes iterable compatibility with web targets that expect the stream chunks to be instances of Uint8 array (encoded text strings).

Previously it was only working on node (somehow!)

(PR also refactors encoding utils to save bundle on text encoder/decoder instances)


Patch:
diff --git a/package.json b/package.json
index 337550f..010842d 100644
--- a/package.json
+++ b/package.json
@@ -61,7 +61,7 @@
     "express": "^4.19.2",
     "get-port": "^7.1.0",
     "get-port-please": "^3.1.2",
-    "h3-nightly": "npm:h3-nightly@2.0.0-1721648175.86e2b90",
+    "h3-nightly": "npm:h3-nightly@2.0.0-1721747307.095de7d",
     "h3-v1": "npm:h3@^1.12.0",
     "jiti": "2.0.0-beta.3",
     "listhen": "^1.7.2",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 993b435..1cefb3a 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -70,8 +70,8 @@ importers:
         specifier: ^3.1.2
         version: 3.1.2
       h3-nightly:
-        specifier: npm:h3-nightly@2.0.0-1721648175.86e2b90
-        version: 2.0.0-1721648175.86e2b90(crossws@0.2.4)
+        specifier: npm:h3-nightly@2.0.0-1721747307.095de7d
+        version: 2.0.0-1721747307.095de7d(crossws@0.2.4)
       h3-v1:
         specifier: npm:h3@^1.12.0
         version: h3@1.12.0
@@ -1693,8 +1693,8 @@ packages:
   graphemer@1.4.0:
     resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==}
 
-  h3-nightly@2.0.0-1721648175.86e2b90:
-    resolution: {integrity: sha512-Ije/JE36OJhhyBP9ikwNufovleC7A7B4kvIHV5/w3u8epxZLeKpSmT/C6B/GgjvMq5UQ+5XltyIl+LKGY3f2XA==}
+  h3-nightly@2.0.0-1721747307.095de7d:
+    resolution: {integrity: sha512-ynd/dcYd/B/OvGl616523U8Tih79xmSZjxC8fgVxGd+k1EGoak0tHPtvgZA1nYsv2xQXUcI5vWB6A2Era8xbyg==}
     peerDependencies:
       crossws: ^0.2.4
     peerDependenciesMeta:
@@ -4797,7 +4797,7 @@ snapshots:
 
   graphemer@1.4.0: {}
 
-  h3-nightly@2.0.0-1721648175.86e2b90(crossws@0.2.4):
+  h3-nightly@2.0.0-1721747307.095de7d(crossws@0.2.4):
     dependencies:
       cookie-es: 1.2.1
       rou3: 0.4.0
diff --git a/src/utils/internal/encoding.ts b/src/utils/internal/encoding.ts
new file mode 100644
index 0000000..6a4ba1b
--- /dev/null
+++ b/src/utils/internal/encoding.ts
@@ -0,0 +1,80 @@
+/**
+Base64 encoding based on https://github.com/denoland/std/tree/main/encoding (modified with url compatibility)
+Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+https://github.com/denoland/std/blob/main/LICENSE
+ */
+
+export const textEncoder = /* @__PURE__ */ new TextEncoder();
+export const textDecoder = /* @__PURE__ */ new TextDecoder();
+
+// ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_
+const base64Code = /* @__PURE__ */ [
+  65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83,
+  84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106,
+  107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121,
+  122, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 45, 95,
+];
+
+export function base64Encode(data: ArrayBuffer | Uint8Array | string): string {
+  const buff = validateBinaryLike(data);
+  if (globalThis.Buffer) {
+    return globalThis.Buffer.from(buff).toString("base64url");
+  }
+  // Credits: https://gist.github.com/enepomnyaschih/72c423f727d395eeaa09697058238727
+  const bytes: number[] = [];
+  let i;
+  const len = buff.length;
+  for (i = 2; i < len; i += 3) {
+    bytes.push(
+      base64Code[buff[i - 2]! >> 2],
+      base64Code[((buff[i - 2]! & 0x03) << 4) | (buff[i - 1]! >> 4)],
+      base64Code[((buff[i - 1]! & 0x0f) << 2) | (buff[i]! >> 6)],
+      base64Code[buff[i]! & 0x3f],
+    );
+  }
+  if (i === len + 1) {
+    // 1 octet yet to write
+    bytes.push(
+      base64Code[buff[i - 2]! >> 2],
+      base64Code[(buff[i - 2]! & 0x03) << 4],
+    );
+  }
+  if (i === len) {
+    // 2 octets yet to write
+    bytes.push(
+      base64Code[buff[i - 2]! >> 2],
+      base64Code[((buff[i - 2]! & 0x03) << 4) | (buff[i - 1]! >> 4)],
+      base64Code[(buff[i - 1]! & 0x0f) << 2],
+    );
+  }
+  // eslint-disable-next-line unicorn/prefer-code-point
+  return String.fromCharCode(...bytes);
+}
+export function base64Decode(b64Url: string): Uint8Array {
+  if (globalThis.Buffer) {
+    return new Uint8Array(globalThis.Buffer.from(b64Url, "base64url"));
+  }
+  const b64 = b64Url.replace(/-/g, "+").replace(/_/g, "/");
+  const binString = atob(b64);
+  const size = binString.length;
+  const bytes = new Uint8Array(size);
+  for (let i = 0; i < size; i++) {
+    // (Uint8Array values are 0-255)
+    // eslint-disable-next-line unicorn/prefer-code-point
+    bytes[i] = binString.charCodeAt(i);
+  }
+  return bytes;
+}
+
+export function validateBinaryLike(source: unknown): Uint8Array {
+  if (typeof source === "string") {
+    return textEncoder.encode(source);
+  } else if (source instanceof Uint8Array) {
+    return source;
+  } else if (source instanceof ArrayBuffer) {
+    return new Uint8Array(source);
+  }
+  throw new TypeError(
+    `The input must be a Uint8Array, a string, or an ArrayBuffer.`,
+  );
+}
diff --git a/src/utils/internal/iron-crypto.ts b/src/utils/internal/iron-crypto.ts
index 8ec7c63..82d6bb2 100644
--- a/src/utils/internal/iron-crypto.ts
+++ b/src/utils/internal/iron-crypto.ts
@@ -6,13 +6,15 @@ https://github.com/brc-dd/iron-webcrypto/blob/v1.2.1/LICENSE.md
 Based on https://github.com/hapijs/iron/tree/v7.0.1
 Copyright (c) 2012-2022, Project contributors Copyright (c) 2012-2020, Sideway Inc All rights reserved.
 https://github.com/hapijs/iron/blob/v7.0.1/LICENSE.md
-
-Base64 encoding based on https://github.com/denoland/std/tree/main/encoding (modified for url compatibility)
-Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
-https://github.com/denoland/std/blob/main/LICENSE
  */
 
 import crypto from "uncrypto"; // Node.js 18 support
+import {
+  base64Decode,
+  base64Encode,
+  textDecoder,
+  textEncoder,
+} from "./encoding";
 
 /** The default encryption and integrity settings. */
 export const defaults: Readonly<SealOptions> = /* @__PURE__ */ Object.freeze({
@@ -162,7 +164,7 @@ export async function hmacWithPassword(
   data: string,
 ): Promise<HMacResult> {
   const key = await generateKey(password, { ...options, hmac: true });
-  const textBuffer = stringToUint8Array(data);
+  const textBuffer = textEncoder.encode(data);
   // prettier-ignore
   const signed = await crypto.subtle.sign({ name: "HMAC" }, key.key, textBuffer);
   const digest = base64Encode(new Uint8Array(signed));
@@ -250,10 +252,10 @@ async function pbkdf2(
   keyLength: number,
   hash: HashAlgorithmIdentifier,
 ): Promise<ArrayBuffer> {
-  const passwordBuffer = stringToUint8Array(password);
+  const passwordBuffer = textEncoder.encode(password);
   // prettier-ignore
   const importedKey = await crypto.subtle.importKey("raw", passwordBuffer, { name: "PBKDF2" }, false, ["deriveBits"]);
-  const saltBuffer = stringToUint8Array(salt);
+  const saltBuffer = textEncoder.encode(salt);
   const params = { name: "PBKDF2", hash, salt: saltBuffer, iterations };
   // prettier-ignore
   const derivation = await crypto.subtle.deriveBits(params, importedKey, keyLength * 8);
@@ -283,7 +285,7 @@ export async function decrypt(
   const decrypted = await crypto.subtle.decrypt(
     ...getEncryptParams(options.algorithm, key, data),
   );
-  return bufferToString(new Uint8Array(decrypted));
+  return textDecoder.decode(decrypted);
 }
 
 function getEncryptParams(
@@ -300,7 +302,7 @@ function getEncryptParams(
         } satisfies AesCtrParams)
       : ({ name: "AES-CBC", iv: key.iv } satisfies AesCbcParams),
     key.key,
-    typeof data === "string" ? stringToUint8Array(data) : data,
+    typeof data === "string" ? textEncoder.encode(data) : data,
   ];
 }
 
@@ -348,91 +350,6 @@ function randomBytes(size: number): Uint8Array {
   return bytes;
 }
 
-// --- encoding ---
-
-const encoder = /* @__PURE__ */ new TextEncoder();
-const decoder = /* @__PURE__ */ new TextDecoder();
-
-function bufferToString(value: Uint8Array): string {
-  return decoder.decode(value);
-}
-
-function stringToUint8Array(value: string): Uint8Array {
-  return encoder.encode(value);
-}
-
-// ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_
-const base64Code = /* @__PURE__ */ [
-  65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83,
-  84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106,
-  107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121,
-  122, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 45, 95,
-];
-
-export function base64Encode(data: ArrayBuffer | Uint8Array | string): string {
-  const buff = validateBinaryLike(data);
-  if (globalThis.Buffer) {
-    return globalThis.Buffer.from(buff).toString("base64url");
-  }
-  // Credits: https://gist.github.com/enepomnyaschih/72c423f727d395eeaa09697058238727
-  const bytes: number[] = [];
-  let i;
-  const len = buff.length;
-  for (i = 2; i < len; i += 3) {
-    bytes.push(
-      base64Code[buff[i - 2]! >> 2],
-      base64Code[((buff[i - 2]! & 0x03) << 4) | (buff[i - 1]! >> 4)],
-      base64Code[((buff[i - 1]! & 0x0f) << 2) | (buff[i]! >> 6)],
-      base64Code[buff[i]! & 0x3f],
-    );
-  }
-  if (i === len + 1) {
-    // 1 octet yet to write
-    bytes.push(
-      base64Code[buff[i - 2]! >> 2],
-      base64Code[(buff[i - 2]! & 0x03) << 4],
-    );
-  }
-  if (i === len) {
-    // 2 octets yet to write
-    bytes.push(
-      base64Code[buff[i - 2]! >> 2],
-      base64Code[((buff[i - 2]! & 0x03) << 4) | (buff[i - 1]! >> 4)],
-      base64Code[(buff[i - 1]! & 0x0f) << 2],
-    );
-  }
-  // eslint-disable-next-line unicorn/prefer-code-point
-  return String.fromCharCode(...bytes);
-}
-export function base64Decode(b64Url: string): Uint8Array {
-  if (globalThis.Buffer) {
-    return new Uint8Array(globalThis.Buffer.from(b64Url, "base64url"));
-  }
-  const b64 = b64Url.replace(/-/g, "+").replace(/_/g, "/");
-  const binString = atob(b64);
-  const size = binString.length;
-  const bytes = new Uint8Array(size);
-  for (let i = 0; i < size; i++) {
-    // (Uint8Array values are 0-255)
-    // eslint-disable-next-line unicorn/prefer-code-point
-    bytes[i] = binString.charCodeAt(i);
-  }
-  return bytes;
-}
-
-export function validateBinaryLike(source: unknown): Uint8Array {
-  if (typeof source === "string") {
-    return encoder.encode(source);
-  } else if (source instanceof Uint8Array) {
-    return source;
-  } else if (source instanceof ArrayBuffer) {
-    return new Uint8Array(source);
-  }
-  throw new TypeError(
-    `The input must be a Uint8Array, a string, or an ArrayBuffer.`,
-  );
-}
-
 // --- Types ---
 
 /** Algorithm used for encryption and decryption. */
diff --git a/src/utils/internal/iterable.ts b/src/utils/internal/iterable.ts
index df9beb6..eb3fdc3 100644
--- a/src/utils/internal/iterable.ts
+++ b/src/utils/internal/iterable.ts
@@ -1,3 +1,5 @@
+import { textEncoder } from "./encoding";
+
 export type IterationSource<Val, Ret = Val> =
   | Iterable<Val>
   | AsyncIterable<Val>
@@ -7,10 +9,9 @@ export type IterationSource<Val, Ret = Val> =
       | Iterator<Val, Ret | undefined>
       | AsyncIterator<Val, Ret | undefined>);
 
-type SendableValue = string | Buffer | Uint8Array;
 export type IteratorSerializer<Value> = (
   value: Value,
-) => SendableValue | undefined;
+) => Uint8Array | undefined;
 
 /**
  * The default implementation for {@link iterable}'s `serializer` argument.
@@ -22,30 +23,25 @@ export type IteratorSerializer<Value> = (
  *
  * @param value - The value to serialize to either a string or Uint8Array.
  */
-export function serializeIterableValue(
-  value: unknown,
-): SendableValue | undefined {
+export function serializeIterableValue(value: unknown): Uint8Array {
   switch (typeof value) {
     case "string": {
-      return value;
+      return textEncoder.encode(value);
     }
     case "boolean":
     case "number":
     case "bigint":
     case "symbol": {
-      return value.toString();
-    }
-    case "function":
-    case "undefined": {
-      return undefined;
+      return textEncoder.encode(value.toString());
     }
     case "object": {
       if (value instanceof Uint8Array) {
         return value;
       }
-      return JSON.stringify(value);
+      return textEncoder.encode(JSON.stringify(value));
     }
   }
+  return new Uint8Array();
 }
 
 export function coerceIterable<V, R>(


Test patch:
diff --git a/test/_setup.ts b/test/_setup.ts
index 1426958..8a234ea 100644
--- a/test/_setup.ts
+++ b/test/_setup.ts
@@ -35,8 +35,12 @@ export function describeMatrix(
     fn(ctx, utils);
   };
   describe(title, () => {
-    run(setupWebTest(opts));
-    run(setupNodeTest(opts));
+    describe("web", () => {
+      run(setupWebTest(opts));
+    });
+    describe("node", () => {
+      run(setupNodeTest(opts));
+    });
   });
 }
 
diff --git a/test/iterable.test.ts b/test/iterable.test.ts
index e031c67..3de4f28 100644
--- a/test/iterable.test.ts
+++ b/test/iterable.test.ts
@@ -4,8 +4,7 @@ import { iterable } from "../src";
 import { describeMatrix } from "./_setup";
 
 describeMatrix("iterable", (t, { it, expect, describe }) => {
-  // TODO: Investigate issue with iterable oon web (Received non-Uint8Array chunk)
-  describe.skipIf(t.target === "web")("iterable", () => {
+  describe("iterable", () => {
     it("sends empty body for an empty iterator", async () => {
       t.app.use((event) => iterable(event, []));
       const result = await t.fetch("/");
@@ -98,7 +97,8 @@ describeMatrix("iterable", (t, { it, expect, describe }) => {
     describe("serializer argument", () => {
       it("is called for every value", async () => {
         const testIterable = [1, "2", { field: 3 }, null];
-        const serializer = vi.fn(() => "x");
+        const textEncoder = new TextEncoder();
+        const serializer = vi.fn(() => textEncoder.encode("x"));
         t.app.use((event) => iterable(event, testIterable, { serializer }));
         const response = await t.fetch("/");
         expect(await response.text()).toBe("x".repeat(testIterable.length));
diff --git a/test/unit/iron-crypto.test.ts b/test/unit/iron-crypto.test.ts
index cca77c5..115fa1b 100644
--- a/test/unit/iron-crypto.test.ts
+++ b/test/unit/iron-crypto.test.ts
@@ -7,6 +7,7 @@ https://github.com/brc-dd/iron-webcrypto/blob/v1.2.1/LICENSE.md
 import { describe, it, assert, expect, vi } from "vitest";
 import { createHmac } from "node:crypto";
 import * as Iron from "../../src/utils/internal/iron-crypto";
+import { base64Encode } from "../../src/utils/internal/encoding";
 
 async function rejects(
   promise: Promise<unknown>,
@@ -29,21 +30,18 @@ describe(`iron crypto`, () => {
     assert.deepEqual(unsealed, obj);
   });
 
-  it(
-    "turns object into a ticket than parses the ticket successfully (no buffer)",
-    { retry: 3 },
-    async () => {
-      vi.stubGlobal("Buffer", undefined);
-      const sealed = await Iron.seal(obj, password, Iron.defaults);
-      const unsealed = await Iron.unseal(
-        sealed,
-        { default: password },
-        Iron.defaults,
-      );
-      vi.unstubAllGlobals();
-      assert.deepEqual(unsealed, obj);
-    },
-  );
+  // TODO: Mocking Buffer makes randomly vite internals fail
+  it.skip("turns object into a ticket than parses the ticket successfully (no buffer)", async () => {
+    vi.stubGlobal("Buffer", undefined);
+    const sealed = await Iron.seal(obj, password, Iron.defaults);
+    const unsealed = await Iron.unseal(
+      sealed,
+      { default: password },
+      Iron.defaults,
+    );
+    vi.unstubAllGlobals();
+    assert.deepEqual(unsealed, obj);
+  });
 
   it("unseal and sealed object with expiration", async () => {
     const options = { ...Iron.defaults, ttl: 200 };
@@ -238,7 +236,7 @@ describe(`iron crypto`, () => {
         const hmac = createHmac(Iron.defaults.integrity.algorithm, key).update(
           data,
         );
-        const digest = Iron.base64Encode(hmac.digest());
+        const digest = base64Encode(hmac.digest());
         const mac = await Iron.hmacWithPassword(
           key,
           Iron.defaults.integrity,
@@ -371,8 +369,8 @@ describe(`iron crypto`, () => {
         Iron.defaults.encryption,
         badJson,
       );
-      const encryptedB64 = Iron.base64Encode(encrypted);
-      const iv = Iron.base64Encode(key.iv);
+      const encryptedB64 = base64Encode(encrypted);
+      const iv = base64Encode(key.iv);
       const macBaseString = `${Iron.macPrefix}**${key.salt}*${iv}*${encryptedB64}*`;
       const mac = await Iron.hmacWithPassword(
         password,
diff --git a/test/unit/iterable.test.ts b/test/unit/iterable.test.ts
index 14c8232..9c0def8 100644
--- a/test/unit/iterable.test.ts
+++ b/test/unit/iterable.test.ts
@@ -4,26 +4,29 @@ import { serializeIterableValue } from "../../src/utils/internal/iterable";
 describe("iterable (unit)", () => {
   describe("serializeIterableValue", () => {
     const exampleDate: Date = new Date(Date.UTC(2015, 6, 21, 3, 24, 54, 888));
+    const encoder = new TextEncoder();
     it.each([
       { value: "Hello, world!", output: "Hello, world!" },
       { value: 123, output: "123" },
       { value: 1n, output: "1" },
       { value: true, output: "true" },
       { value: false, output: "false" },
-      { value: undefined, output: undefined },
+      { value: undefined, output: "" },
       { value: null, output: "null" },
       { value: exampleDate, output: JSON.stringify(exampleDate) },
       { value: { field: 1 }, output: '{"field":1}' },
       { value: [1, 2, 3], output: "[1,2,3]" },
-      { value: () => {}, output: undefined },
+      { value: () => {}, output: "" },
       {
-        value: Buffer.from("Hello, world!"),
-        output: Buffer.from("Hello, world!"),
+        value: encoder.encode("Hello, world!"),
+        output: encoder.encode("Hello, world!"),
       },
       { value: Uint8Array.from([1, 2, 3]), output: Uint8Array.from([1, 2, 3]) },
     ])("$value => $output", ({ value, output }) => {
       const serialized = serializeIterableValue(value);
-      expect(serialized).toStrictEqual(output);
+      expect(serialized).toStrictEqual(
+        output instanceof Uint8Array ? output : encoder.encode(output),
+      );
     });
   });
 });