Repository: endojs/endo
Pull request #2294 (2024-05-21 21:13:43)

feat(bundle-source): Zip original sources with --no-transforms
Closes: #2295 
Refs: #400, #2252

## Description

This change adds a mode to the `bundle-source` command with the initial flag `--no-transforms` that generates “endo zip base64” style bundles without applying the module-to-program transform and SES shim censorship evasion transforms, such that the original files on disk appear in the zip file. This is a preparatory step, necessary for building test artifacts, in advance of full support for this bundle style.

### Security Considerations

`bundle-source` is part of the Endo and Agoric toolkit and it, or its surrogate, participate in the toolchain for generating content that can be confined by Hardened JavaScript, but is not trusted by Hardened JavaScript at runtime. It does however _currently_ run with all the authority of the developer in their development environment and its integrity must be carefully guarded.

### Scaling Considerations

No improvements expected at this time, but in pursuit of #400, it may be possible to move the heavy and performance sensitive JavaScript transform components from `bundle-source` to `import-bundle` and only suffer the performance cost of these transforms on Node.js, where those costs are more readily born by some runtimes. Precompiled bundles may continue to be the preferred medium for deployment to the web, for example.

### Documentation Considerations

We will need to advertise the `--no-transforms` flag eventually, since there will be a period where it is advisable if not necessary to generate contracts and caplets targeting the XS runtime.

### Testing Considerations

I have included a test that verifies the API behavior and manually run the following to verify behavior for the CLI:

```sh
rm -rf bundles
yarn bundle-source --no-transforms --cache-json bundles demo/circular/a.js circular-a
rm -rf circular-a
mkdir -p circular-a
jq -r .endoZipBase64 bundles/bundle-circular-a.json | base64 -d > circular-a/circular-a.zip
(cd circular-a; unzip circular-a.zip)
jq . circular-a/compartment-map.json
# verifying the final module entires have parser: 'mjs'
```

### Compatibility Considerations

This flag is opt-in and breaks no prior behaviors. This introduces a new entry to the build cache meta-data and may cause some bundles to be regenerated one extra time after upgrading.

### Upgrade Considerations

This should not impact upgrade, though it participates in the greater #400 story which will require xsnap upgrades to come to bear.

kriskowal: Looking for review buddies for work pursuant to XS native compartments, #400. This change touches `bundle-source` code I’ve shared maintainership with @dckc and compartment-mapper which has benefited tremendously from contributions from @naugtur and @boneskull, built on substantial work by @michaelfig. I would like to page @gibson042 into Compartment Mapper since it touches upon work we propose to TC39.

Patch:
diff --git a/packages/bundle-source/cache.js b/packages/bundle-source/cache.js
index ddfa89641..4f380e769 100644
--- a/packages/bundle-source/cache.js
+++ b/packages/bundle-source/cache.js
@@ -16,6 +16,7 @@ const { Fail, quote: q } = assert;
  * @property {string} bundleFileName
  * @property {string} bundleTime ISO format
  * @property {number} bundleSize
+ * @property {boolean} noTransforms
  * @property {{ relative: string, absolute: string }} moduleSource
  * @property {Array<{ relativePath: string, mtime: string, size: number }>} contents
  */
@@ -41,9 +42,18 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     ...bundleOptions
   } = opts || {};
 
-  const add = async (rootPath, targetName, log = defaultLog) => {
+  /**
+   * @param {string} rootPath
+   * @param {string} targetName
+   * @param {Logger} [log]
+   * @param {object} [options]
+   * @param {boolean} [options.noTransforms]
+   */
+  const add = async (rootPath, targetName, log = defaultLog, options = {}) => {
     const srcRd = cwd.neighbor(rootPath);
 
+    const { noTransforms = false } = options;
+
     const statsByPath = new Map();
 
     const loggedRead = async loc => {
@@ -69,10 +79,14 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     const bundleWr = wr.neighbor(bundleFileName);
     const metaWr = wr.neighbor(toBundleMeta(targetName));
 
-    const bundle = await bundleSource(rootPath, bundleOptions, {
-      ...readPowers,
-      read: loggedRead,
-    });
+    const bundle = await bundleSource(
+      rootPath,
+      { ...bundleOptions, noTransforms },
+      {
+        ...readPowers,
+        read: loggedRead,
+      },
+    );
 
     const { moduleFormat } = bundle;
     assert.equal(moduleFormat, 'endoZipBase64');
@@ -98,6 +112,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
           size,
         }),
       ),
+      noTransforms,
     };
 
     await metaWr.atomicWriteText(JSON.stringify(meta, null, 2));
@@ -200,9 +215,16 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
    * @param {string} rootPath
    * @param {string} targetName
    * @param {Logger} [log]
+   * @param {object} [options]
+   * @param {boolean} [options.noTransforms]
    * @returns {Promise<BundleMeta>}
    */
-  const validateOrAdd = async (rootPath, targetName, log = defaultLog) => {
+  const validateOrAdd = async (
+    rootPath,
+    targetName,
+    log = defaultLog,
+    options = {},
+  ) => {
     const metaText = await loadMetaText(targetName, log);
 
     /** @type {BundleMeta | undefined} */
@@ -211,7 +233,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     if (meta !== undefined) {
       try {
         meta = await validate(targetName, rootPath, log, meta);
-        const { bundleTime, bundleSize, contents } = meta;
+        const { bundleTime, bundleSize, contents, noTransforms } = meta;
         log(
           `${wr}`,
           toBundleName(targetName),
@@ -221,6 +243,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
           bundleTime,
           'with size',
           bundleSize,
+          noTransforms ? 'w/o transforms' : 'with transforms',
         );
       } catch (invalid) {
         meta = undefined;
@@ -230,8 +253,8 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
 
     if (meta === undefined) {
       log(`${wr}`, 'add:', targetName, 'from', rootPath);
-      meta = await add(rootPath, targetName, log);
-      const { bundleFileName, bundleTime, contents } = meta;
+      meta = await add(rootPath, targetName, log, options);
+      const { bundleFileName, bundleTime, contents, noTransforms } = meta;
       log(
         `${wr}`,
         'bundled',
@@ -240,6 +263,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
         bundleFileName,
         'at',
         bundleTime,
+        noTransforms ? 'w/o transforms' : 'with transforms',
       );
     }
 
@@ -251,11 +275,14 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
    * @param {string} rootPath
    * @param {string} [targetName]
    * @param {Logger} [log]
+   * @param {object} [options]
+   * @param {boolean} [options.noTransforms]
    */
   const load = async (
     rootPath,
     targetName = readPowers.basename(rootPath, '.js'),
     log = defaultLog,
+    options = {},
   ) => {
     const found = loaded.get(targetName);
     // console.log('load', { targetName, found: !!found, rootPath });
@@ -264,7 +291,7 @@ export const makeBundleCache = (wr, cwd, readPowers, opts) => {
     }
     const todo = makePromiseKit();
     loaded.set(targetName, { rootPath, bundle: todo.promise });
-    const bundle = await validateOrAdd(rootPath, targetName, log)
+    const bundle = await validateOrAdd(rootPath, targetName, log, options)
       .then(
         ({ bundleFileName }) =>
           import(`${wr.readOnly().neighbor(bundleFileName)}`),
@@ -298,12 +325,12 @@ export const makeNodeBundleCache = async (
   nonce,
 ) => {
   const [fs, path, url, crypto, timers, os] = await Promise.all([
-    await loadModule('fs'),
-    await loadModule('path'),
-    await loadModule('url'),
-    await loadModule('crypto'),
-    await loadModule('timers'),
-    await loadModule('os'),
+    loadModule('fs'),
+    loadModule('path'),
+    loadModule('url'),
+    loadModule('crypto'),
+    loadModule('timers'),
+    loadModule('os'),
   ]);
 
   if (nonce === undefined) {
diff --git a/packages/bundle-source/package.json b/packages/bundle-source/package.json
index 93b401906..a867ab6a4 100644
--- a/packages/bundle-source/package.json
+++ b/packages/bundle-source/package.json
@@ -39,8 +39,10 @@
   "devDependencies": {
     "@endo/lockdown": "^1.0.7",
     "@endo/ses-ava": "^1.2.2",
+    "@endo/zip": "^1.0.5",
     "ava": "^6.1.3",
     "c8": "^7.14.0",
+    "eslint": "^8.57.0",
     "typescript": "5.5.0-beta"
   },
   "keywords": [],
diff --git a/packages/bundle-source/src/main.js b/packages/bundle-source/src/main.js
index 813d1194e..8115ffe57 100644
--- a/packages/bundle-source/src/main.js
+++ b/packages/bundle-source/src/main.js
@@ -1,9 +1,31 @@
 // @ts-check
+import { parseArgs } from 'util';
 import { jsOpts, jsonOpts, makeNodeBundleCache } from '../cache.js';
 
 const USAGE =
   'bundle-source [--cache-js | --cache-json] cache/ module1.js bundleName1 module2.js bundleName2 ...';
 
+const options = /** @type {const} */ ({
+  'no-transforms': {
+    type: 'boolean',
+    short: 'T',
+    multiple: false,
+  },
+  'cache-js': {
+    type: 'string',
+    multiple: false,
+  },
+  'cache-json': {
+    type: 'string',
+    multiple: false,
+  },
+  // deprecated
+  to: {
+    type: 'string',
+    multiple: false,
+  },
+});
+
 /**
  * @param {[to: string, dest: string, ...rest: string[]]} args
  * @param {object} powers
@@ -13,20 +35,42 @@ const USAGE =
  * @returns {Promise<void>}
  */
 export const main = async (args, { loadModule, pid, log }) => {
-  const [to, dest, ...pairs] = args;
-  if (!(dest && pairs.length > 0 && pairs.length % 2 === 0)) {
+  const {
+    values: {
+      'no-transforms': noTransforms,
+      'cache-json': cacheJson,
+      'cache-js': cacheJs,
+      // deprecated
+      to: cacheJsAlias,
+    },
+    positionals: pairs,
+  } = parseArgs({ args, options, allowPositionals: true });
+
+  if (
+    !(
+      pairs.length > 0 &&
+      pairs.length % 2 === 0 &&
+      [cacheJson, cacheJs, cacheJsAlias].filter(Boolean).length === 1
+    )
+  ) {
     throw Error(USAGE);
   }
 
+  /** @type {string} */
+  let dest;
   let cacheOpts;
   // `--to` option is deprecated, but we now use it to mean `--cache-js`.
-  if (to === '--to') {
+  if (cacheJs !== undefined) {
+    dest = cacheJs;
     cacheOpts = jsOpts;
-  } else if (to === '--cache-js') {
+  } else if (cacheJsAlias !== undefined) {
+    dest = cacheJsAlias;
     cacheOpts = jsOpts;
-  } else if (to === '--cache-json') {
+  } else if (cacheJson !== undefined) {
+    dest = cacheJson;
     cacheOpts = jsonOpts;
   } else {
+    // unreachable
     throw Error(USAGE);
   }
 
@@ -41,6 +85,8 @@ export const main = async (args, { loadModule, pid, log }) => {
     const [bundleRoot, bundleName] = pairs.slice(ix, ix + 2);
 
     // eslint-disable-next-line no-await-in-loop
-    await cache.validateOrAdd(bundleRoot, bundleName);
+    await cache.validateOrAdd(bundleRoot, bundleName, undefined, {
+      noTransforms,
+    });
   }
 };
diff --git a/packages/bundle-source/src/types.js b/packages/bundle-source/src/types.js
index 94d3ac881..07bb8d4df 100644
--- a/packages/bundle-source/src/types.js
+++ b/packages/bundle-source/src/types.js
@@ -64,6 +64,9 @@ export {};
  * @property {T} [format]
  * @property {boolean} [dev] - development mode, for test bundles that need
  * access to devDependencies of the entry package.
+ * @property {boolean} [noTransforms] - when true, generates a bundle with the
+ * original sources instead of SES-shim specific ESM and CJS. This may become
+ * default in a future major version.
  */
 
 /**
diff --git a/packages/bundle-source/src/zip-base64.js b/packages/bundle-source/src/zip-base64.js
index b81102f49..81a903c45 100644
--- a/packages/bundle-source/src/zip-base64.js
+++ b/packages/bundle-source/src/zip-base64.js
@@ -7,7 +7,10 @@ import url from 'url';
 import fs from 'fs';
 import os from 'os';
 
-import { makeAndHashArchive } from '@endo/compartment-mapper/archive.js';
+import { defaultParserForLanguage as transformingParserForLanguage } from '@endo/compartment-mapper/archive-parsers.js';
+import { defaultParserForLanguage as transparentParserForLanguage } from '@endo/compartment-mapper/import-parsers.js';
+import { mapNodeModules } from '@endo/compartment-mapper/node-modules.js';
+import { makeAndHashArchiveFromMap } from '@endo/compartment-mapper/archive-lite.js';
 import { encodeBase64 } from '@endo/base64';
 import { whereEndoCache } from '@endo/where';
 import { makeReadPowers } from '@endo/compartment-mapper/node-powers.js';
@@ -17,12 +20,31 @@ const textEncoder = new TextEncoder();
 const textDecoder = new TextDecoder();
 const readPowers = makeReadPowers({ fs, url, crypto });
 
+/**
+ * @param {string} startFilename
+ * @param {object} [options]
+ * @param {boolean} [options.dev]
+ * @param {boolean} [options.cacheSourceMaps]
+ * @param {boolean} [options.noTransforms]
+ * @param {Record<string, string>} [options.commonDependencies]
+ * @param {object} [grantedPowers]
+ * @param {(bytes: string | Uint8Array) => string} [grantedPowers.computeSha512]
+ * @param {typeof import('path)['resolve']} [grantedPowers.pathResolve]
+ * @param {typeof import('os')['userInfo']} [grantedPowers.userInfo]
+ * @param {typeof process['env']} [grantedPowers.env]
+ * @param {typeof process['platform']} [grantedPowers.platform]
+ */
 export async function bundleZipBase64(
   startFilename,
   options = {},
   grantedPowers = {},
 ) {
-  const { dev = false, cacheSourceMaps = false, commonDependencies } = options;
+  const {
+    dev = false,
+    cacheSourceMaps = false,
+    noTransforms = false,
+    commonDependencies,
+  } = options;
   const powers = { ...readPowers, ...grantedPowers };
   const {
     computeSha512,
@@ -137,9 +159,11 @@ export async function bundleZipBase64(
     return { bytes: objectBytes, parser, sourceMap };
   };
 
-  const { bytes, sha512 } = await makeAndHashArchive(powers, entry, {
-    dev,
-    moduleTransforms: {
+  let parserForLanguage = transparentParserForLanguage;
+  let moduleTransforms = {};
+  if (!noTransforms) {
+    parserForLanguage = transformingParserForLanguage;
+    moduleTransforms = {
       async mjs(
         sourceBytes,
         specifier,
@@ -170,12 +194,25 @@ export async function bundleZipBase64(
           sourceMap,
         );
       },
-    },
-    sourceMapHook(sourceMap, sourceDescriptor) {
-      sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor));
-    },
+    };
+  }
+
+  const compartmentMap = await mapNodeModules(powers, entry, {
+    dev,
     commonDependencies,
   });
+
+  const { bytes, sha512 } = await makeAndHashArchiveFromMap(
+    powers,
+    compartmentMap,
+    {
+      parserForLanguage,
+      moduleTransforms,
+      sourceMapHook(sourceMap, sourceDescriptor) {
+        sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor));
+      },
+    },
+  );
   assert(sha512);
   await Promise.all(sourceMapJobs);
   const endoZipBase64 = encodeBase64(bytes);
diff --git a/yarn.lock b/yarn.lock
index 847133a33..761dcfd3b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -190,11 +190,13 @@ __metadata:
     "@endo/promise-kit": "npm:^1.1.2"
     "@endo/ses-ava": "npm:^1.2.2"
     "@endo/where": "npm:^1.0.5"
+    "@endo/zip": "npm:^1.0.5"
     "@rollup/plugin-commonjs": "npm:^19.0.0"
     "@rollup/plugin-node-resolve": "npm:^13.0.0"
     acorn: "npm:^8.2.4"
     ava: "npm:^6.1.3"
     c8: "npm:^7.14.0"
+    eslint: "npm:^8.57.0"
     rollup: "npm:^2.79.1"
     typescript: "npm:5.5.0-beta"
   bin:


Test patch:
diff --git a/packages/bundle-source/test/no-transforms.test.js b/packages/bundle-source/test/no-transforms.test.js
new file mode 100644
index 000000000..6079534b6
--- /dev/null
+++ b/packages/bundle-source/test/no-transforms.test.js
@@ -0,0 +1,36 @@
+// @ts-check
+import test from '@endo/ses-ava/prepare-endo.js';
+
+import fs from 'fs';
+import url from 'url';
+import { decodeBase64 } from '@endo/base64';
+import { ZipReader } from '@endo/zip';
+import bundleSource from '../src/index.js';
+
+test('no-transforms applies no transforms', async t => {
+  const entryPath = url.fileURLToPath(
+    new URL(`../demo/circular/a.js`, import.meta.url),
+  );
+  const { endoZipBase64 } = await bundleSource(entryPath, {
+    moduleFormat: 'endoZipBase64',
+    noTransforms: true,
+  });
+  const endoZipBytes = decodeBase64(endoZipBase64);
+  const zipReader = new ZipReader(endoZipBytes);
+  const compartmentMapBytes = zipReader.read('compartment-map.json');
+  const compartmentMapText = new TextDecoder().decode(compartmentMapBytes);
+  const compartmentMap = JSON.parse(compartmentMapText);
+  const { entry, compartments } = compartmentMap;
+  const compartment = compartments[entry.compartment];
+  const module = compartment.modules[entry.module];
+  // Alleged module type is not precompiled (pre-mjs-json)
+  t.is(module.parser, 'mjs');
+
+  const moduleBytes = zipReader.read(
+    `${compartment.location}/${module.location}`,
+  );
+  const moduleText = new TextDecoder().decode(moduleBytes);
+  const originalModuleText = await fs.promises.readFile(entryPath, 'utf-8');
+  // And, just to be sure, the text in the bundle matches the original text.
+  t.is(moduleText, originalModuleText);
+});