Repository: endojs/endo
Pull request #2352 (2024-07-13 05:06:40)

feat(compartment-mapper): JSON module support for makeBundle
Refs: #400, https://github.com/Agoric/agoric-sdk/issues/9686

## Description

Endo’s zip bundle format and Rollup both support importing JSON modules. This change adds JSON module support to Endo’s script bundle format. We need this to have Rollup parity for Agoric Core Eval scripts and also to unblock test262 for XS native compartment parity tests.

### Security Considerations

Obviates Rollup, increases legibility of script bundles.

### Scaling Considerations

None.

### Documentation Considerations

None. JSON module support is assumed. No one will notice unless it doesn’t work.

### Testing Considerations

This change adds a JSON module to the existing bundle test fixture. This also verifies parity with the Zip bundle format.

### Compatibility Considerations

This make Endo more compatible with Rollup, but Endo continues to not support `import / with { type: "json" }`, which would be necessary for parity with Node.js proper and newer bundlers.

### Upgrade Considerations

None.

kriskowal: Ironed out the types.

Patch:
diff --git a/packages/compartment-mapper/NEWS.md b/packages/compartment-mapper/NEWS.md
index 9c0387e18..d98df7d92 100644
--- a/packages/compartment-mapper/NEWS.md
+++ b/packages/compartment-mapper/NEWS.md
@@ -21,6 +21,7 @@ User-visible changes to the compartment mapper:
   For example, `import-parsers.js` does not entrain Babel.
   The new `import-lite.js` does not entrain `node-modules.js` and composes
   with potential alternative package discovery, storage, and locks.
+- Adds JSON module support to `makeBundle`.
 
 # 0.9.0 (2023-08-07)
 
diff --git a/packages/compartment-mapper/src/bundle-cjs.js b/packages/compartment-mapper/src/bundle-cjs.js
index 25dbb6432..22decc7fb 100644
--- a/packages/compartment-mapper/src/bundle-cjs.js
+++ b/packages/compartment-mapper/src/bundle-cjs.js
@@ -1,5 +1,10 @@
 /* Provides CommonJS support for `bundle.js`. */
-// @ts-nocheck
+
+/** @import {VirtualModuleSource} from 'ses' */
+/** @import {BundlerSupport} from './bundle.js' */
+
+/** @typedef {VirtualModuleSource & {cjsFunctor: string}} CjsModuleSource */
+
 /** quotes strings */
 const q = JSON.stringify;
 
@@ -15,7 +20,8 @@ const exportsCellRecord = exportsList =>
   );
 
 // This function is serialized and references variables from its destination scope.
-const runtime = function wrapCjsFunctor(num) {
+const runtime = `\
+function wrapCjsFunctor(num) {
   /* eslint-disable no-undef */
   return ({ imports = {} }) => {
     const moduleCells = cells[num];
@@ -50,8 +56,9 @@ const runtime = function wrapCjsFunctor(num) {
     moduleCells['*'].set(Object.freeze(starExports));
   };
   /* eslint-enable no-undef */
-}.toString();
+}`;
 
+/** @type {BundlerSupport<CjsModuleSource>} */
 export default {
   runtime,
   getBundlerKit({
diff --git a/packages/compartment-mapper/src/bundle-json.js b/packages/compartment-mapper/src/bundle-json.js
new file mode 100644
index 000000000..72ea3dadb
--- /dev/null
+++ b/packages/compartment-mapper/src/bundle-json.js
@@ -0,0 +1,24 @@
+/* Provides JSON support for `bundle.js`. */
+
+const textDecoder = new TextDecoder();
+
+export default {
+  runtime: '',
+  getBundlerKit({ index, indexedImports, bytes }) {
+    // Round-trip to revalidate JSON and squeeze out space.
+    const json = JSON.stringify(JSON.parse(textDecoder.decode(bytes)));
+    return {
+      getFunctor: () => `\
+// === functors[${index}] ===
+(set) => set(${json}),
+`,
+      getCells: () => `\
+    { default: cell('default') },
+`,
+      getReexportsWiring: () => '',
+      getFunctorCall: () => `\
+  functors[${index}](cells[${index}].default.set);
+`,
+    };
+  },
+};
diff --git a/packages/compartment-mapper/src/bundle-mjs.js b/packages/compartment-mapper/src/bundle-mjs.js
index 28cd8591a..83aa7ed38 100644
--- a/packages/compartment-mapper/src/bundle-mjs.js
+++ b/packages/compartment-mapper/src/bundle-mjs.js
@@ -1,5 +1,8 @@
 /* Provides ESM support for `bundle.js`. */
 
+/** @import {PrecompiledModuleSource} from 'ses' */
+/** @import {BundlerSupport} from './bundle.js' */
+
 /** quotes strings */
 const q = JSON.stringify;
 
@@ -47,6 +50,7 @@ function observeImports(map, importName, importIndex) {
 }
 `;
 
+/** @type {BundlerSupport<PrecompiledModuleSource>} */
 export default {
   runtime,
   getBundlerKit({
diff --git a/packages/compartment-mapper/src/bundle.js b/packages/compartment-mapper/src/bundle.js
index 4c99b7e92..bbee7f336 100644
--- a/packages/compartment-mapper/src/bundle.js
+++ b/packages/compartment-mapper/src/bundle.js
@@ -1,6 +1,8 @@
 // @ts-check
 /* eslint no-shadow: 0 */
 
+/** @import {StaticModuleType} from 'ses' */
+
 /** @import {ArchiveOptions} from './types.js' */
 /** @import {CompartmentDescriptor} from './types.js' */
 /** @import {CompartmentSources} from './types.js' */
@@ -11,6 +13,56 @@
 /** @import {Sources} from './types.js' */
 /** @import {WriteFn} from './types.js' */
 
+/**
+ * @typedef {object} BundlerKit
+ * @property {() => string} getFunctor Produces a JavaScript string consisting of
+ * a function expression followed by a comma delimiter that will be evaluated in
+ * a lexical scope with no free variables except the globals.
+ * In the generated bundle runtime, the function will receive an environment
+ * record: a record mapping every name of the corresponding module's internal
+ * namespace to a "cell" it can use to get, set, or observe the linked
+ * variable.
+ * @property {() => string} getCells Produces a JavaScript string consisting of
+ * a JavaScript object and a trailing comma.
+ * The string is evaluated in a lexical context with a `cell` maker, the `cells`
+ * array of every module's internal environment record.
+ * @property {() => string} getFunctorCall Produces a JavaScript string may
+ * be a statement that calls this module's functor with the calling convention
+ * appropriate for its language, injecting whatever cells it needs to link to
+ * other module namespaces.
+ * @property {() => string} getReexportsWiring Produces a JavaScript string
+ * that may include statements that bind the cells reexported by this module.
+ */
+
+/**
+ * @template {unknown} SpecificModuleSource
+ * @typedef {object} BundleModule
+ * @property {string} key
+ * @property {string} compartmentName
+ * @property {string} moduleSpecifier
+ * @property {string} parser
+ * @property {StaticModuleType & SpecificModuleSource} record
+ * @property {Record<string, string>} resolvedImports
+ * @property {Record<string, number>} indexedImports
+ * @property {Uint8Array} bytes
+ * @property {number} index
+ * @property {BundlerKit} bundlerKit
+ */
+
+/**
+ * @template {unknown} SpecificModuleSource
+ * @callback GetBundlerKit
+ * @param {BundleModule<SpecificModuleSource>} module
+ * @returns {BundlerKit}
+ */
+
+/**
+ * @template {unknown} SpecificModuleSource
+ * @typedef {object} BundlerSupport
+ * @property {string} runtime
+ * @property {GetBundlerKit<SpecificModuleSource>} getBundlerKit
+ */
+
 import { resolve } from './node-module-specifier.js';
 import { compartmentMapForNodeModules } from './node-modules.js';
 import { search } from './search.js';
@@ -22,6 +74,7 @@ import { parseLocatedJson } from './json.js';
 
 import mjsSupport from './bundle-mjs.js';
 import cjsSupport from './bundle-cjs.js';
+import jsonSupport from './bundle-json.js';
 
 const textEncoder = new TextEncoder();
 
@@ -39,8 +92,11 @@ const sortedModules = (
   entryCompartmentName,
   entryModuleSpecifier,
 ) => {
+  /** @type {BundleModule<unknown>[]} */
   const modules = [];
+  /** @type {Map<string, string>} aliaes */
   const aliases = new Map();
+  /** @type {Set<string>} seen */
   const seen = new Set();
 
   /**
@@ -56,7 +112,9 @@ const sortedModules = (
 
     const source = compartmentSources[compartmentName][moduleSpecifier];
     if (source !== undefined) {
-      const { record, parser, deferredError } = source;
+      const { record, parser, deferredError, bytes } = source;
+      assert(parser !== undefined);
+      assert(bytes !== undefined);
       if (deferredError) {
         throw Error(
           `Cannot bundle: encountered deferredError ${deferredError}`,
@@ -84,6 +142,11 @@ const sortedModules = (
           parser,
           record,
           resolvedImports,
+          bytes,
+          // @ts-expect-error
+          index: undefined,
+          // @ts-expect-error
+          bundlerKit: null,
         });
 
         return key;
@@ -117,31 +180,37 @@ const sortedModules = (
   return { modules, aliases };
 };
 
-const implementationPerParser = {
+/** @type {Record<string, BundlerSupport<unknown>>} */
+const bundlerSupportForLanguage = {
   'pre-mjs-json': mjsSupport,
   'pre-cjs-json': cjsSupport,
+  json: jsonSupport,
 };
 
-function getRuntime(parser) {
-  return implementationPerParser[parser]
-    ? implementationPerParser[parser].runtime
-    : `/*unknown parser:${parser}*/`;
-}
-
-function getBundlerKitForModule(module) {
-  const parser = module.parser;
-  if (!implementationPerParser[parser]) {
-    const warning = `/*unknown parser:${parser}*/`;
+/** @param {string} language */
+const getRuntime = language =>
+  bundlerSupportForLanguage[language]
+    ? bundlerSupportForLanguage[language].runtime
+    : `/*unknown language:${language}*/`;
+
+/** @param {BundleModule<unknown>} module */
+const getBundlerKitForModule = module => {
+  const language = module.parser;
+  assert(language !== undefined);
+  if (bundlerSupportForLanguage[language] === undefined) {
+    const warning = `/*unknown language:${language}*/`;
     // each item is a function to avoid creating more in-memory copies of the source text etc.
+    /** @type {BundlerKit} */
     return {
-      getFunctor: () => `(()=>{${warning}})`,
-      getCells: `{${warning}}`,
-      getFunctorCall: warning,
+      getFunctor: () => `(()=>{${warning}}),`,
+      getCells: () => `{${warning}},`,
+      getFunctorCall: () => warning,
+      getReexportsWiring: () => '',
     };
   }
-  const { getBundlerKit } = implementationPerParser[parser];
+  const { getBundlerKit } = bundlerSupportForLanguage[language];
   return getBundlerKit(module);
-}
+};
 
 /**
  * @param {ReadFn | ReadPowers | MaybeReadPowers} readPowers
@@ -266,10 +335,7 @@ export const makeBundle = async (readPowers, moduleLocation, options) => {
 
   const bundle = `\
 'use strict';
-(() => {
-  const functors = [
-${''.concat(...modules.map(m => m.bundlerKit.getFunctor()))}\
-]; // functors end
+(functors => {
 
   const cell = (name, value = undefined) => {
     const observers = [];
@@ -297,7 +363,7 @@ ${''.concat(...modules.map(m => m.bundlerKit.getCells()))}\
 
 ${''.concat(...modules.map(m => m.bundlerKit.getReexportsWiring()))}\
 
-const namespaces = cells.map(cells => Object.freeze(Object.create(null, {
+  const namespaces = cells.map(cells => Object.freeze(Object.create(null, {
     ...cells,
     // Make this appear like an ESM module namespace object.
     [Symbol.toStringTag]: {
@@ -317,7 +383,7 @@ ${''.concat(...Array.from(parsersInUse).map(parser => getRuntime(parser)))}
 ${''.concat(...modules.map(m => m.bundlerKit.getFunctorCall()))}\
 
   return cells[cells.length - 1]['*'].get();
-})();
+})([${''.concat(...modules.map(m => m.bundlerKit.getFunctor()))}]);
 `;
 
   return bundle;
diff --git a/packages/compartment-mapper/src/types.js b/packages/compartment-mapper/src/types.js
index 64df74e2f..c10fd28f2 100644
--- a/packages/compartment-mapper/src/types.js
+++ b/packages/compartment-mapper/src/types.js
@@ -1,12 +1,11 @@
 // @ts-check
-/// <reference types="ses"/>
 
 export {};
 
 /** @import {FinalStaticModuleType} from 'ses' */
-/** @import {ThirdPartyStaticModuleInterface} from 'ses' */
 /** @import {ImportHook} from 'ses' */
 /** @import {StaticModuleType} from 'ses' */
+/** @import {ThirdPartyStaticModuleInterface} from 'ses' */
 /** @import {Transform} from 'ses' */
 
 // /////////////////////////////////////////////////////////////////////////////
diff --git a/packages/ses/types.d.ts b/packages/ses/types.d.ts
index b60aef135..4d40fc801 100644
--- a/packages/ses/types.d.ts
+++ b/packages/ses/types.d.ts
@@ -47,6 +47,7 @@ export type ModuleExportsNamespace = Record<string, any>;
 
 export type __LiveExportMap__ = Record<string, [string, boolean]>;
 export type __FixedExportMap__ = Record<string, [string]>;
+export type __ReexportMap__ = Record<string, Array<[string, string]>>;
 
 export interface PrecompiledModuleSource {
   imports: Array<string>;
@@ -55,6 +56,7 @@ export interface PrecompiledModuleSource {
   __syncModuleProgram__: string;
   __liveExportMap__: __LiveExportMap__;
   __fixedExportMap__: __FixedExportMap__;
+  __reexportMap__: __ReexportMap__;
 }
 
 export interface VirtualModuleSource {


Test patch:
diff --git a/packages/compartment-mapper/test/bundle.test.js b/packages/compartment-mapper/test/bundle.test.js
index b8e61b69a..2fcf01e69 100644
--- a/packages/compartment-mapper/test/bundle.test.js
+++ b/packages/compartment-mapper/test/bundle.test.js
@@ -52,6 +52,7 @@ const expectedLog = [
     fromMjs: 'foo',
     zzz: 1,
   }),
+  { widdershins: 'againshins' },
 ];
 
 test('bundles work', async t => {
diff --git a/packages/compartment-mapper/test/fixtures-0/node_modules/bundle/json.json b/packages/compartment-mapper/test/fixtures-0/node_modules/bundle/json.json
new file mode 100644
index 000000000..eb28c5701
--- /dev/null
+++ b/packages/compartment-mapper/test/fixtures-0/node_modules/bundle/json.json
@@ -0,0 +1,1 @@
+{"widdershins": "againshins"}
diff --git a/packages/compartment-mapper/test/fixtures-0/node_modules/bundle/main.js b/packages/compartment-mapper/test/fixtures-0/node_modules/bundle/main.js
index 6fa0c2cb4..4f6b8e024 100644
--- a/packages/compartment-mapper/test/fixtures-0/node_modules/bundle/main.js
+++ b/packages/compartment-mapper/test/fixtures-0/node_modules/bundle/main.js
@@ -30,4 +30,7 @@ print(colors);
 import 'bundle-dep';
 
 import * as cjsstuff from './icando.cjs';
-print(cjsstuff)
+print(cjsstuff);
+
+import json from './json.json';
+print(json);