Repository: danvk/literate-ts (ts)

Issue:
Allow inline replacements
Replacements have to be defined in external files right now. It would be more convenient to be able to define them inline in the same file (presumably in a commented-out block). There could be either a naming convention (`foo-replacement`) or a compiler directive (`replace:foo`).

Pull request description:
Inline replacements
Fixes #96 

I refactored the replacement logic a bit and settled on a new directive, `replace-with-id`. This gives you lots of flexibility and allows a replacement to be used with multiple code samples.

Patch:
diff --git a/README.md b/README.md
index 377e5a0..b2a069f 100644
--- a/README.md
+++ b/README.md
@@ -230,6 +230,10 @@ See above for how to give directive to literate-ts in your source format.
     Start prepending a sample from earlier in the source file. Useful to reestablish some
     context after a `reset`.
   </dd>
+  <dt>verifier:replace-with-id:ID</dt>
+  <dd>
+    Replace this code sample with another one. The two samples have to match in a very precise way for this to work, see [replacements](#replacements).
+  </dd>
   <dt>verifier:tsconfig:setting=value</dt>
   <dd>
     Set a tsconfig setting for the next code sample, e.g. `strictNullChecks=false`.
@@ -291,6 +295,8 @@ addition to `COMPRESS...END`, you can also use `HIDE...END` to completely remove
 be careful not to mislead the reader when you do this.
 (This syntax comes from [pyliterate][pylit-post].)
 
+If you'd like to keep your source files self-contained, you can put the replacement in a code block and reference it with a `replace-with-id` directive. See [this sample file](/examples/asciidoc/failures.asciidoc) for examples.
+
 ## Command-Line Options
 
 - `--help`: Show help.
diff --git a/examples/asciidoc/failures.asciidoc b/examples/asciidoc/failures.asciidoc
index 88ff29c..93e701b 100644
--- a/examples/asciidoc/failures.asciidoc
+++ b/examples/asciidoc/failures.asciidoc
@@ -168,3 +168,35 @@ You'll need to run this sample with `-r examples/asciidoc/replacements` to see t
 ----
 Inline sample does not match sample in source file
 ----
+
+You can also define replacements inline in your file, presumably hidden away in a comment. This is done using the `replace-with-id` directive. Here's an example of that in action.
+
+// verifier:replace-with-id:calculate-age-replacement
+[source,ts]
+----
+function calculateAge(birthDate: Date): number {
+  // ...
+}
+----
+
+////
+[[calculate-age-replacement]]
+[source,ts]
+----
+function calculateAge(birthDate: Date): number {
+  // COMPRESS
+  return 0;
+  // END
+}
+----
+////
+
+And here's an example of that failing to match. Note that the replacement and the original may be defined in either order.
+
+// verifier:replace-with-id:calculate-age-replacement
+[source,ts]
+----
+function calculateAge(birthDate: string): number {
+  // ...
+}
+----
diff --git a/src/code-sample.ts b/src/code-sample.ts
index 4688893..925f7cd 100644
--- a/src/code-sample.ts
+++ b/src/code-sample.ts
@@ -33,6 +33,7 @@ function process(
   let skipRemaining = false;
   let prependNext = false;
   let prependLines: number[] | null = null;
+  let nextIsReplaced: string | null = null;
   let nodeModules: readonly string[] = [];
   let tsOptions: {[key: string]: string | boolean} = {};
   let nextIsTSX = false;
@@ -65,7 +66,7 @@ function process(
       } else if (directive.startsWith('prepend-id-to-following')) {
         prefixes = prefixes.concat([
           {
-            id: directive.split(':').slice(1).join(':'),
+            id: directive.split(':', 2)[1],
           },
         ]);
       } else if (directive.startsWith('skip')) {
@@ -84,6 +85,8 @@ function process(
         nextShouldCheckJs = true;
         tsOptions['allowJs'] = true; // convenience, it's useless without this!
         tsOptions['noEmit'] = true;
+      } else if (directive.startsWith('replace-with-id')) {
+        nextIsReplaced = directive.split(':', 2)[1];
       } else {
         throw new Error(`Unknown directive: ${directive}`);
       }
@@ -110,6 +113,7 @@ function process(
             language: lastLanguage as CodeSample['language'],
             content,
             prefixes,
+            replacementId: nextIsReplaced ?? undefined,
             nodeModules,
             isTSX: nextIsTSX,
             checkJS: nextShouldCheckJs,
@@ -135,6 +139,7 @@ function process(
       skipNext = false;
       nextIsTSX = false;
       nextShouldCheckJs = false;
+      nextIsReplaced = null;
     },
   };
 
@@ -181,10 +186,8 @@ ${strippedSource.trim()}
   return true;
 }
 
-export function applyPrefixes(
-  samples: PrefixedCodeSample[],
-  sources: {[id: string]: string} = {},
-): CodeSample[] {
+/** Combine a code sample with all active prefixes to form a standalone code sample. */
+export function applyPrefixes(samples: PrefixedCodeSample[]): CodeSample[] {
   const idToSample = _.keyBy(samples, 'id');
   const sliceLines = (text: string, lines: number[] | undefined) =>
     lines
@@ -194,10 +197,13 @@ export function applyPrefixes(
           .join('\n')
       : text;
   return samples.map(sample => {
+    if (sample.replacementId) {
+      throw new Error(`Logic error: sample {sample.id} was not replaced.`);
+    }
     const prefixes = sample.id.endsWith('-output') ? [] : sample.prefixes;
     const content = prefixes
-      .map(({id, lines}) => sliceLines(sources[id] || idToSample[id].content, lines))
-      .concat([sources[sample.id] || sample.content])
+      .map(({id, lines}) => sliceLines(idToSample[id].content, lines))
+      .concat([sample.content])
       .join('\n');
     return {
       descriptor: sample.descriptor,
@@ -214,6 +220,44 @@ export function applyPrefixes(
   });
 }
 
+export function applyReplacements(
+  rawSamples: readonly Readonly<PrefixedCodeSample>[],
+  externalReplacements: {[id: string]: string} = {},
+): PrefixedCodeSample[] {
+  const samples = _.cloneDeep(rawSamples) as PrefixedCodeSample[];
+  const idToSample = _.keyBy(samples, 'id');
+
+  // First check the external replacements
+  for (const sample of samples) {
+    const {id} = sample;
+    const source = externalReplacements[id];
+    if (source) {
+      checkSource(sample, source);
+      sample.content = source;
+    }
+  }
+  // TODO: flag unused replacements
+
+  // Next do the inline replacements
+  for (const sample of samples) {
+    const {replacementId} = sample;
+    if (!replacementId) {
+      continue;
+    }
+
+    const replacementSample = idToSample[replacementId];
+    if (!replacementSample) {
+      fail(`No sample with ID ${replacementId} to replace ${sample.id}.`);
+    } else {
+      checkSource(sample, replacementSample.content);
+      sample.content = replacementSample.content;
+      delete sample.replacementId;
+    }
+  }
+
+  return samples;
+}
+
 export function extractSamples(text: string, slug: string, sourceFile: string) {
   let processor;
   if (sourceFile.endsWith('.asciidoc')) {
diff --git a/src/main.ts b/src/main.ts
index 26a3109..9392fdc 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -3,12 +3,12 @@ import path, {isAbsolute} from 'path';
 
 import chalk from 'chalk';
 import glob from 'fast-glob';
-import _ from 'lodash';
+import _, {replace} from 'lodash';
 import ora from 'ora';
 import ts from 'typescript';
 import yargs from 'yargs';
 
-import {checkSource, applyPrefixes, extractSamples} from './code-sample.js';
+import {checkSource, applyPrefixes, extractSamples, applyReplacements} from './code-sample.js';
 import {startLog, log, flushLog, logFile} from './logger.js';
 import {runNode} from './node-runner.js';
 import {
@@ -32,7 +32,7 @@ const argv = yargs(process.argv.slice(2))
       type: 'string',
       description:
         'If specified, load **/*.{ts,js,txt} under this directory' +
-        'as additional sources. See README for details on how to use these.',
+        'as additional sources. See README.md for details on how to use these.',
       alias: 'r',
     },
     filter: {
@@ -177,14 +177,8 @@ async function processSourceFile(path: string, fileNum: number, outOf: number) {
   const rawSamples = extractSamples(text, fileSlug(path), path);
   log(`Found ${rawSamples.length} code samples in ${path}`);
 
-  for (const sample of rawSamples) {
-    const {id} = sample;
-    const source = sources[id];
-    if (source) {
-      checkSource(sample, source);
-    }
-  }
-  const samples = applyPrefixes(rawSamples, sources);
+  const replacedSamples = applyReplacements(rawSamples, sources);
+  const samples = applyPrefixes(replacedSamples);
 
   const outputs = _.keyBy(samples, 'id');
   for (const [i, sample] of Object.entries(samples)) {
diff --git a/src/types.ts b/src/types.ts
index d42ba2d..583f7da 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -26,6 +26,7 @@ export interface Prefix {
 
 export interface PrefixedCodeSample extends CodeSample {
   prefixes: readonly Prefix[];
+  replacementId?: string;
 }
 
 export interface SampleOutput {


Test patch:
diff --git a/src/test/__snapshots__/asciidoc.test.ts.snap b/src/test/__snapshots__/asciidoc.test.ts.snap
index 074070c..6fbbffc 100644
--- a/src/test/__snapshots__/asciidoc.test.ts.snap
+++ b/src/test/__snapshots__/asciidoc.test.ts.snap
@@ -13,6 +13,7 @@ exports[`extractSamples snapshot: backticks 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "backticks.asciidoc",
     "tsOptions": {},
@@ -26,6 +27,7 @@ exports[`extractSamples snapshot: backticks 1`] = `
     "language": "tsx",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "backticks.asciidoc",
     "tsOptions": {},
@@ -39,6 +41,7 @@ exports[`extractSamples snapshot: backticks 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "backticks.asciidoc",
     "tsOptions": {},
@@ -59,6 +62,7 @@ exports[`extractSamples snapshot: commented-sample 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "commented-sample.asciidoc",
     "tsOptions": {},
@@ -84,6 +88,7 @@ val  // Type is number | null",
         "id": "commented-sample-6",
       },
     ],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "commented-sample.asciidoc",
     "tsOptions": {},
@@ -104,6 +109,7 @@ exports[`extractSamples snapshot: doc1 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "doc1.asciidoc",
     "tsOptions": {},
@@ -111,6 +117,61 @@ exports[`extractSamples snapshot: doc1 1`] = `
 ]
 `;
 
+exports[`extractSamples snapshot: inline-replacement 1`] = `
+[
+  {
+    "checkJS": false,
+    "content": "function calculateAge(birthDate: Date): number {
+  // ...
+}",
+    "descriptor": "./inline-replacement.asciidoc:5",
+    "id": "inline-replacement-5",
+    "isTSX": false,
+    "language": "ts",
+    "nodeModules": [],
+    "prefixes": [],
+    "replacementId": "calculate-age-replacement",
+    "sectionHeader": null,
+    "sourceFile": "inline-replacement.asciidoc",
+    "tsOptions": {},
+  },
+  {
+    "checkJS": false,
+    "content": "function calculateAge(birthDate: Date): number {
+  // COMPRESS
+  return 0;
+  // END
+}",
+    "descriptor": "./inline-replacement.asciidoc:11",
+    "id": "calculate-age-replacement",
+    "isTSX": false,
+    "language": "ts",
+    "nodeModules": [],
+    "prefixes": [],
+    "replacementId": undefined,
+    "sectionHeader": null,
+    "sourceFile": "inline-replacement.asciidoc",
+    "tsOptions": {},
+  },
+  {
+    "checkJS": false,
+    "content": "function calculateAge(birthDate: string): number {
+  // ...
+}",
+    "descriptor": "./inline-replacement.asciidoc:27",
+    "id": "inline-replacement-27",
+    "isTSX": false,
+    "language": "ts",
+    "nodeModules": [],
+    "prefixes": [],
+    "replacementId": "calculate-age-replacement",
+    "sectionHeader": null,
+    "sourceFile": "inline-replacement.asciidoc",
+    "tsOptions": {},
+  },
+]
+`;
+
 exports[`extractSamples snapshot: multilinetype 1`] = `
 [
   {
@@ -126,6 +187,7 @@ exports[`extractSamples snapshot: multilinetype 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "multilinetype.asciidoc",
     "tsOptions": {},
@@ -143,6 +205,7 @@ exports[`extractSamples snapshot: multilinetype 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "multilinetype.asciidoc",
     "tsOptions": {},
@@ -161,6 +224,7 @@ exports[`extractSamples snapshot: noid 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "noid.asciidoc",
     "tsOptions": {},
@@ -179,6 +243,7 @@ exports[`extractSamples snapshot: prepend 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend.asciidoc",
     "tsOptions": {},
@@ -196,6 +261,7 @@ exports[`extractSamples snapshot: prepend 1`] = `
         "id": "prefix",
       },
     ],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend.asciidoc",
     "tsOptions": {},
@@ -209,6 +275,7 @@ exports[`extractSamples snapshot: prepend 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend.asciidoc",
     "tsOptions": {},
@@ -227,6 +294,7 @@ exports[`extractSamples snapshot: prepend-multiple 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend-multiple.asciidoc",
     "tsOptions": {},
@@ -244,6 +312,7 @@ exports[`extractSamples snapshot: prepend-multiple 1`] = `
         "id": "prepend-multiple-3",
       },
     ],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend-multiple.asciidoc",
     "tsOptions": {},
@@ -264,6 +333,7 @@ exports[`extractSamples snapshot: prepend-multiple 1`] = `
         "id": "prepend-multiple-9",
       },
     ],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend-multiple.asciidoc",
     "tsOptions": {},
diff --git a/src/test/__snapshots__/markdown.test.ts.snap b/src/test/__snapshots__/markdown.test.ts.snap
index 6e8b8d8..552054e 100644
--- a/src/test/__snapshots__/markdown.test.ts.snap
+++ b/src/test/__snapshots__/markdown.test.ts.snap
@@ -11,6 +11,7 @@ exports[`markdown should match snapshots: commented-sample 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "commented-sample.md",
     "tsOptions": {},
@@ -36,6 +37,7 @@ val  // Type is number | null",
         "id": "commented-sample-5",
       },
     ],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "commented-sample.md",
     "tsOptions": {},
@@ -56,6 +58,7 @@ exports[`markdown should match snapshots: doc1 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "doc1.md",
     "tsOptions": {},
@@ -70,6 +73,7 @@ exports[`markdown should match snapshots: doc1 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "doc1.md",
     "tsOptions": {},
@@ -92,6 +96,7 @@ exports[`markdown should match snapshots: multilinetype 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "multilinetype.md",
     "tsOptions": {},
@@ -109,6 +114,7 @@ exports[`markdown should match snapshots: multilinetype 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "multilinetype.md",
     "tsOptions": {},
@@ -127,6 +133,7 @@ exports[`markdown should match snapshots: noid 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "noid.md",
     "tsOptions": {},
@@ -145,6 +152,7 @@ exports[`markdown should match snapshots: prepend 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend.md",
     "tsOptions": {},
@@ -162,6 +170,7 @@ exports[`markdown should match snapshots: prepend 1`] = `
         "id": "prefix",
       },
     ],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend.md",
     "tsOptions": {},
@@ -175,6 +184,7 @@ exports[`markdown should match snapshots: prepend 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend.md",
     "tsOptions": {},
@@ -193,6 +203,7 @@ exports[`markdown should match snapshots: prepend-multiple 1`] = `
     "language": "ts",
     "nodeModules": [],
     "prefixes": [],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend-multiple.md",
     "tsOptions": {},
@@ -210,6 +221,7 @@ exports[`markdown should match snapshots: prepend-multiple 1`] = `
         "id": "prepend-multiple-2",
       },
     ],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend-multiple.md",
     "tsOptions": {},
@@ -230,6 +242,7 @@ exports[`markdown should match snapshots: prepend-multiple 1`] = `
         "id": "prepend-multiple-7",
       },
     ],
+    "replacementId": undefined,
     "sectionHeader": null,
     "sourceFile": "prepend-multiple.md",
     "tsOptions": {},
diff --git a/src/test/code-sample.test.ts b/src/test/code-sample.test.ts
index ab17fb6..6293912 100644
--- a/src/test/code-sample.test.ts
+++ b/src/test/code-sample.test.ts
@@ -1,6 +1,7 @@
-import {stripSource, applyPrefixes, extractSamples} from '../code-sample.js';
+import {stripSource, applyPrefixes, extractSamples, applyReplacements} from '../code-sample.js';
+import {PrefixedCodeSample} from '../types.js';
 import {dedent} from '../utils.js';
-import {baseSample} from './common.js';
+import {baseExtract, baseSample} from './common.js';
 
 const ASCIIDOC_PREPEND = `
 
@@ -313,3 +314,77 @@ describe('stripSource', () => {
       const x = foo();`);
   });
 });
+
+describe('applyReplacements', () => {
+  const foo: PrefixedCodeSample = {
+    ...baseExtract,
+    language: 'ts',
+    descriptor: './source.asciidoc:4',
+    id: 'foo',
+    content: '',
+  };
+
+  const compressed = dedent`
+  function foo() {
+    // ...
+  }
+  const x = foo();`;
+  const full = dedent`
+  function foo() {
+    // COMPRESS
+    return 1 + 2 + 3;
+    // END
+  }
+  const x = foo();`;
+
+  it('should apply external replacements', () => {
+    expect(
+      applyReplacements(
+        [
+          {
+            ...foo,
+            content: compressed,
+          },
+        ],
+        {
+          foo: full,
+        },
+      ),
+    ).toEqual([
+      {
+        ...foo,
+        content: full,
+      },
+    ]);
+  });
+
+  it('should apply inline replacements', () => {
+    expect(
+      applyReplacements(
+        [
+          {
+            ...foo,
+            content: compressed,
+            replacementId: 'replace',
+          },
+          {
+            ...foo,
+            id: 'replace',
+            content: full,
+          },
+        ],
+        {},
+      ),
+    ).toEqual([
+      {
+        ...foo,
+        content: full,
+      },
+      {
+        ...foo,
+        id: 'replace',
+        content: full,
+      },
+    ]);
+  });
+});
diff --git a/src/test/inputs/inline-replacement.asciidoc b/src/test/inputs/inline-replacement.asciidoc
new file mode 100644
index 0000000..ce10486
--- /dev/null
+++ b/src/test/inputs/inline-replacement.asciidoc
@@ -0,0 +1,31 @@
+You can also define replacements inline in your file, presumably hidden away in a comment. This is done using the `replace-with-id` directive. Here's an example of that in action.
+
+// verifier:replace-with-id:calculate-age-replacement
+[source,ts]
+----
+function calculateAge(birthDate: Date): number {
+  // ...
+}
+----
+
+////
+[[calculate-age-replacement]]
+[source,ts]
+----
+function calculateAge(birthDate: Date): number {
+  // COMPRESS
+  return 0;
+  // END
+}
+----
+////
+
+And here's an example of that failing to match. Note that the replacement and the original may be defined in either order.
+
+// verifier:replace-with-id:calculate-age-replacement
+[source,ts]
+----
+function calculateAge(birthDate: string): number {
+  // ...
+}
+----