Repository: danvk/literate-ts (ts)

Issue:
Improve error message when replacement does not match code sample
It would be helpful if the error message included _which_ code sample didn't match, at the very least!

```
$ time yarn verify
yarn run v1.22.19
$ literate-ts -r ../../github/internal-effective-typescript/checks *.asciidoc
Logging details to /var/folders/t_/3xnk295j79v51cmlqvtnhslc0000gn/T/tmp-36835gLlKWjIZ4dMU/log.txt
Verifying with TypeScript 5.1.3
⠋ 2/9: ./ch02-types.asciidoc: 20/157 ./ch02-types.asciidoc:381
./ch02-types.asciidoc:381: Failed type assertion for type PointKeys = keyof Point; (tested PointKeys)
⠙ 2/9: ./ch02-types.asciidoc: 110/157 ./ch02-types.asciidoc:2264
./ch02-types.asciidoc:2264: Failed type assertion for type OptionsKeys = keyof Options; (tested OptionsKeys)
⠋ 3/9: ./ch03-inference.asciidoc
undefined: Inline sample does not match sample in source file
TypeError: Cannot read properties of undefined (reading 'descriptor')
    at Object.fail (/Users/danvk/Dropbox/effective-typescript/node_modules/literate-ts/dist/src/test-tracker.js:44:44)
    at Object.checkSource (/Users/danvk/Dropbox/effective-typescript/node_modules/literate-ts/dist/src/code-sample.js:148:24)
    at /Users/danvk/Dropbox/effective-typescript/node_modules/literate-ts/dist/src/main.js:163:31
    at Generator.next (<anonymous>)
    at /Users/danvk/Dropbox/effective-typescript/node_modules/literate-ts/dist/src/main.js:8:71
    at new Promise (<anonymous>)
    at __awaiter (/Users/danvk/Dropbox/effective-typescript/node_modules/literate-ts/dist/src/main.js:4:12)
    at processSourceFile (/Users/danvk/Dropbox/effective-typescript/node_modules/literate-ts/dist/src/main.js:152:12)
    at Object.<anonymous> (/Users/danvk/Dropbox/effective-typescript/node_modules/literate-ts/dist/src/main.js:185:19)
    at Generator.next (<anonymous>)
✨  Done in 41.98s.
yarn verify  60.71s user 3.97s system 153% cpu 42.169 total
```
Add an on-disk cache for sample results
A cache would be very, very effective at speeding up repeated runs and would greatly facilitate running this via a VS Code extension (#44).

The cache key would be:

- hash of fully-resolved code sample
- compiler options
- {literate-ts, typescript, node} version

The cache value would be:

- num matched/mismatched errors / type assertions
- output of running the code (if it was run)

Pull request description:
Add an on-disk cache
Verifying chapter 1 of _Effective TypeScript_, uncached:

    ✓ All 60 samples passed!
    ✨  Done in 13.76s.

Verifying cached:

    ✓ All 60 samples passed!
    ✨  Done in 0.24s.

`:sparkles:` indeed!

This adds a `--nocache` flag to bypass the cache, which is set to live in `/tmp/literate-ts-cache` with this PR (this will change in #64).

Fixes #62 
Fixes #55


Patch:
diff --git a/.eslintrc.js b/.eslintrc.js
index 97c3fb2..c1efe56 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -11,5 +11,6 @@ module.exports =  {
   rules: {
     '@typescript-eslint/explicit-function-return-type': 'off',
     '@typescript-eslint/no-non-null-assertion': 'off',
+    '@typescript-eslint/no-use-before-define': 'off',
   }
 };
diff --git a/package.json b/package.json
index a1b240e..c778c0e 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
   },
   "dependencies": {
     "chalk": "^2.4.2",
+    "fast-json-stable-stringify": "^2.1.0",
     "fs-extra": "^8.1.0",
     "glob": "^7.1.4",
     "lodash": "^4.17.15",
diff --git a/src/code-sample.ts b/src/code-sample.ts
index 9d6ddf4..1c48492 100644
--- a/src/code-sample.ts
+++ b/src/code-sample.ts
@@ -6,6 +6,7 @@ import {fail} from './test-tracker';
 import {extractAsciidocSamples} from './asciidoc';
 import {extractMarkdownSamples} from './markdown';
 import {generateIdMetadata} from './metadata';
+import {dedent} from './utils';
 
 export interface Processor {
   setLineNum(line: number): void;
@@ -167,12 +168,18 @@ export function checkSource(sample: CodeSample, source: string) {
   // Strip out code behind HIDE..END markers
   const strippedSource = stripSource(source);
   if (sample.content.trim() !== strippedSource.trim()) {
-    fail('Inline sample does not match sample in source file', sample);
-    log('Inline sample:');
-    log(sample.content.trim());
-    log('----');
-    log('Stripped source file sample:');
-    log(strippedSource.trim() + '\n');
+    fail(
+      `
+Inline sample for ${sample.id} does not match sample in source file
+Inline sample:
+${sample.content.trim()}
+----
+Stripped source file sample:
+${strippedSource.trim()}
+----
+      `.trimStart(),
+      sample,
+    );
     return false;
   }
   return true;
diff --git a/src/main.ts b/src/main.ts
index b2e1f8b..2455c95 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -21,13 +21,8 @@ import {
 } from './test-tracker';
 import {checkTs, ConfigBundle} from './ts-checker';
 import {CodeSample} from './types';
-import {getTempDir, writeTempFile, fileSlug} from './utils';
-
-const packagePath = path.join(
-  __dirname,
-  // The path to package.json is slightly different when you run via ts-node
-  __dirname.includes('dist') ? '../../package.json' : '../package.json',
-);
+import {writeTempFile, fileSlug} from './utils';
+import {VERSION} from './version';
 
 const argv = yargs
   .strict()
@@ -49,13 +44,14 @@ const argv = yargs
       type: 'boolean',
       description: 'Log to stderr in addition to a log file',
     },
+    nocache: {
+      type: 'boolean',
+      description: `Don't read previous results from cache.`,
+    },
   })
   .version(
     'version',
-    [
-      `literate-ts version: ${require(packagePath).version}`,
-      `TypeScript version: ${ts.version}`,
-    ].join('\n'),
+    [`literate-ts version: ${VERSION}`, `TypeScript version: ${ts.version}`].join('\n'),
   )
   .parse();
 
@@ -93,7 +89,7 @@ function checkOutput(expectedOutput: string, input: CodeSample) {
   }
 
   // Remove stack traces from output
-  const tmpDir = getTempDir();
+  const tmpDir = path.dirname(actualOutput.path);
   const checkOutput = (actualOutput.stderr + actualOutput.stdout)
     .split('\n')
     .filter(line => !line.startsWith('    at ')) // prune stack traces to one line
@@ -117,6 +113,8 @@ function checkOutput(expectedOutput: string, input: CodeSample) {
     log('Actual:');
     log(checkOutput);
     log('----');
+  } else {
+    log('Actual output matched expected.');
   }
 }
 
@@ -126,7 +124,12 @@ async function checkSample(sample: CodeSample, idToSample: {[id: string]: CodeSa
   startSample(sample);
 
   if (language === 'ts' || (language === 'js' && sample.checkJS)) {
-    await checkTs(sample, id + '-output' in idToSample, typeScriptBundle);
+    const result = await checkTs(sample, id + '-output' in idToSample, typeScriptBundle, {
+      skipCache: !!argv.nocache,
+    });
+    if (result.output !== undefined) {
+      sample.output = result.output;
+    }
   } else if (language === 'js') {
     // Run the sample through Node and record the output.
     const path = writeTempFile(id + '.js', content);
@@ -147,7 +150,6 @@ async function checkSample(sample: CodeSample, idToSample: {[id: string]: CodeSa
       fail(`No paired input: #${inputId}`);
     } else {
       checkOutput(content, input);
-      log('Actual output matched expected.');
     }
   }
   finishSample();
diff --git a/src/node-runner.ts b/src/node-runner.ts
index 5e89f00..ad51995 100644
--- a/src/node-runner.ts
+++ b/src/node-runner.ts
@@ -7,17 +7,18 @@ export interface ExecErrorType {
   code: number;
   stdout: string;
   stderr: string;
+  path: string;
 }
 
 /** Run a JavaScript program through Node.js. Returns the output. */
 export async function runNode(path: string): Promise<ExecErrorType> {
   try {
     const {stdout, stderr} = await util.promisify(exec)(`node ${path}`);
-    return {code: 0, stderr, stdout};
+    return {code: 0, stderr, stdout, path};
   } catch (eIn) {
     const e = eIn as ExecErrorType;
     const {code, stderr, stdout} = e;
     log(`Node exited with error ${e.code} on ${path}`);
-    return {code, stderr, stdout};
+    return {code, stderr, stdout, path};
   }
 }
diff --git a/src/ts-checker.ts b/src/ts-checker.ts
index aaa7085..9711661 100644
--- a/src/ts-checker.ts
+++ b/src/ts-checker.ts
@@ -1,14 +1,15 @@
 import fs from 'fs-extra';
 
-import _ from 'lodash';
+import stableJsonStringify from 'fast-json-stable-stringify';
+import _, {VERSION} from 'lodash';
 import path from 'path';
 import ts from 'typescript';
 
 import {log} from './logger';
-import {fail} from './test-tracker';
-import {writeTempFile, matchAndExtract, getTempDir, matchAll} from './utils';
+import {fail, getLastFailReason} from './test-tracker';
+import {writeTempFile, matchAndExtract, getTempDir, matchAll, sha256, tuple} from './utils';
 import {CodeSample} from './types';
-import {runNode} from './node-runner';
+import {ExecErrorType, runNode} from './node-runner';
 
 export interface TypeScriptError {
   line: number;
@@ -345,11 +346,44 @@ function getNodeAtPosition(sourceFile: ts.SourceFile, position: number): ts.Node
   return candidate;
 }
 
-function matchModuloWhitespace(actual: string, expected: string): boolean {
+/** Normalize "B | A" -> "A | B" */
+export function sortUnions(type: string): string {
+  let level = 0;
+  const topLevelPipes: number[] = [];
+  for (let i = 0; i < type.length; i++) {
+    const c = type.charAt(i);
+    if (c === '|' && level === 0) {
+      topLevelPipes.push(i);
+    } else if (c === '{' || c === '(' || c === '<') {
+      level += 1;
+    } else if (c === '}' || c === '}' || c === '>') {
+      level -= 1;
+    }
+    // TODO: quotes
+  }
+
+  if (level !== 0 || topLevelPipes.length === 0) {
+    return type; // do no harm if we don't understand the type.
+  }
+  topLevelPipes.splice(0, 0, -1);
+  const parts = topLevelPipes.map((idx, i) => type.slice(idx + 1, topLevelPipes[i + 1]).trim());
+  return _.sortBy(parts).join(' | ');
+}
+
+export function matchModuloWhitespace(actual: string, expected: string): boolean {
   // TODO: it's much easier to normalize actual based on the displayParts
   //       This isn't 100% correct if a type has a space in it, e.g. type T = "string literal"
-  const normActual = actual.replace(/[\n\r ]+/g, ' ').trim();
-  const normExpected = expected.replace(/[\n\r ]+/g, ' ').trim();
+  const normalize = (input: string) => {
+    const [name, type] = input.split(/[:=]/, 2);
+    const normType = sortUnions(type)
+      .replace(/[\n\r ]+/g, ' ')
+      .replace(/\( */g, '(')
+      .replace(/ *\)/, ')')
+      .trim();
+    return `${name}: ${normType}`;
+  };
+  const normActual = normalize(actual);
+  const normExpected = normalize(expected);
   return normActual === normExpected;
 }
 
@@ -438,8 +472,63 @@ export function getLanguageServiceHost(program: ts.Program): ts.LanguageServiceH
   };
 }
 
+export interface CheckTsResult {
+  passed: boolean;
+  failReason?: string;
+  output?: ExecErrorType;
+  // TODO: include more details about errors
+}
+
+function getCheckTsCacheKey(inSample: CodeSample, runCode: boolean) {
+  const {descriptor: _1, id: _2, sectionHeader: _3, sourceFile: _4, ...sample} = inSample;
+  const key = {
+    sample,
+    runCode,
+    // `config` has compiler options but these are already in `sample`
+    versions: {
+      typeScript: ts.version,
+      literateTs: VERSION,
+      nodeJs: process.version,
+    },
+  };
+  return tuple(sha256(stableJsonStringify(key)), key);
+}
+
+export async function checkTs(
+  sample: CodeSample,
+  runCode: boolean,
+  config: ConfigBundle,
+  options: {skipCache: boolean},
+): Promise<CheckTsResult> {
+  const [key, cacheKey] = getCheckTsCacheKey(sample, runCode);
+  const tempFilePath = `/tmp/literate-ts-cache/${key}.json`;
+  const hit = await fs.pathExists(tempFilePath);
+  if (hit && !options.skipCache) {
+    const result = await fs.readFile(tempFilePath, 'utf8');
+    const {key: _, ...out} = JSON.parse(result) as CheckTsResult & {key: unknown};
+    if (out.failReason) {
+      fail(out.failReason);
+    }
+    return out;
+  }
+
+  const result = await uncachedCheckTs(sample, runCode, config);
+  if (result.passed === false) {
+    result.failReason = getLastFailReason() ?? undefined;
+  }
+
+  if (!options.skipCache) {
+    await fs.writeFile(tempFilePath, JSON.stringify({...result, key: cacheKey}), 'utf8');
+  }
+  return result;
+}
+
 /** Verify that a TypeScript sample has the expected errors and no others. */
-export async function checkTs(sample: CodeSample, runCode: boolean, config: ConfigBundle) {
+async function uncachedCheckTs(
+  sample: CodeSample,
+  runCode: boolean,
+  config: ConfigBundle,
+): Promise<CheckTsResult> {
   const {id, content} = sample;
   const fileName = id + (sample.isTSX ? '.tsx' : `.${sample.language}`);
   const tsFile = writeTempFile(fileName, content);
@@ -469,7 +558,7 @@ export async function checkTs(sample: CodeSample, runCode: boolean, config: Conf
     if (!match) {
       fail(`Could not find requested node_module ${nodeModule}. See logs for details.`);
       log('Looked in:\n  ' + candidates.join('\n  '));
-      return;
+      return {passed: false};
     }
     fs.copySync(match, path.join(nodeModulesPath, nodeModule));
   }
@@ -486,7 +575,7 @@ export async function checkTs(sample: CodeSample, runCode: boolean, config: Conf
   const source = program.getSourceFile(tsFile);
   if (!source) {
     fail('Unable to load TS source file');
-    return;
+    return {passed: false};
   }
 
   let diagnostics = ts.getPreEmitDiagnostics(program);
@@ -497,7 +586,7 @@ export async function checkTs(sample: CodeSample, runCode: boolean, config: Conf
 
     if (emitResult.emitSkipped) {
       fail('Failed to emit JavaScript for TypeScript sample.');
-      return;
+      return {passed: false};
     }
   }
 
@@ -548,13 +637,14 @@ export async function checkTs(sample: CodeSample, runCode: boolean, config: Conf
     log(`tsconfig options: ${JSON.stringify(options)}`);
   }
 
-  if (!runCode) return;
+  if (!runCode) return {passed: ok};
 
   const jsFile = tsFile.replace(/\.ts$/, '.js');
   if (!fs.existsSync(jsFile)) {
     fail(`Did not produce JS output in expected place: ${jsFile}`);
-    return;
+    return {passed: false};
   }
 
-  sample.output = await runNode(jsFile);
+  const output = await runNode(jsFile);
+  return {passed: false, output};
 }
diff --git a/src/types.ts b/src/types.ts
index 64d0185..d42ba2d 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -32,4 +32,5 @@ export interface SampleOutput {
   code: number;
   stdout: string;
   stderr: string;
+  path: string;
 }
diff --git a/src/utils.ts b/src/utils.ts
index db682bb..f1fa1a6 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -1,3 +1,4 @@
+import {createHash} from 'crypto';
 import fs from 'fs';
 import tmp from 'tmp';
 import {basename} from 'path';
@@ -73,3 +74,11 @@ export function dedent(strings: TemplateStringsArray, ...values: (string | numbe
   }
   return fullString;
 }
+
+export function sha256(message: string) {
+  return createHash('sha256')
+    .update(message)
+    .digest('hex');
+}
+
+export const tuple = <Args extends unknown[]>(...args: Args): Args => args;
diff --git a/src/version.ts b/src/version.ts
new file mode 100644
index 0000000..2fe89bc
--- /dev/null
+++ b/src/version.ts
@@ -0,0 +1,10 @@
+import path from 'path';
+
+// TODO: use https://www.npmjs.com/package/read-pkg-up instead
+const packagePath = path.join(
+  __dirname,
+  // The path to package.json is slightly different when you run via ts-node
+  __dirname.includes('dist') ? '../../package.json' : '../package.json',
+);
+
+export const VERSION = require(packagePath).version;


Test patch:
diff --git a/src/test-tracker.ts b/src/test-tracker.ts
index 4b1d6ce..796ea58 100644
--- a/src/test-tracker.ts
+++ b/src/test-tracker.ts
@@ -29,23 +29,30 @@ export function finishSample() {
   const elapsedMs = Date.now() - sampleStartMs;
   log(`\nEND #${currentSample!.descriptor} (${elapsedMs} ms)\n`);
   currentSample = undefined;
+  lastFailReason = null;
 }
 
+let lastFailReason: string | null = null;
 export function fail(message: string, sample?: CodeSample) {
   if (sample === undefined) {
     sample = currentSample;
   }
+  lastFailReason = message;
 
-  const fullMessage = `💥 ${currentSample?.descriptor}: ${message}`;
+  const fullMessage = `💥 ${sample?.descriptor}: ${message}`;
   if (!isLoggingToStderr()) {
     console.error('\n' + fullMessage);
   }
   log(fullMessage);
   if (!(global as any).__TEST__) {
-    results[currentFile][currentSample!.descriptor]++;
+    results[currentFile][sample!.descriptor]++;
   }
 }
 
+export function getLastFailReason(): string | null {
+  return lastFailReason;
+}
+
 export function getTestResults() {
   return results;
 }
diff --git a/src/test/ts-checker.test.ts b/src/test/ts-checker.test.ts
index d388a90..5c4be08 100644
--- a/src/test/ts-checker.test.ts
+++ b/src/test/ts-checker.test.ts
@@ -8,6 +8,8 @@ import {
   extractTypeAssertions,
   checkTypeAssertions,
   getLanguageServiceHost,
+  matchModuloWhitespace,
+  sortUnions,
 } from '../ts-checker';
 import {dedent} from '../utils';
 
@@ -491,4 +493,39 @@ describe('ts-checker', () => {
     //   `)).toBe(true);
     // });
   });
+
+  describe('matchModuloWhitespace', () => {
+    it('should normalize whitespace around parens', () => {
+      const expected =
+        'type T = <T extends object, K extends keyof T>( obj: T, ...keys: K[] ) => Pick<T, K>';
+      const actual =
+        'type T = <T extends object, K extends keyof T>(obj: T, ...keys: K[]) => Pick<T, K>';
+      expect(matchModuloWhitespace(actual, expected)).toBe(true);
+    });
+
+    it('should flag different types', () => {
+      expect(matchModuloWhitespace('const x: string', 'const x: number')).toBe(false);
+    });
+
+    it('should allow differing orders for unions', () => {
+      expect(matchModuloWhitespace('const ab: B | A', 'const ab: A|B')).toBe(true);
+    });
+  });
+
+  describe('sortUnions', () => {
+    it('should sort unions', () => {
+      expect(sortUnions('C | B | A')).toEqual('A | B | C');
+      expect(sortUnions('string | Date | number')).toEqual('Date | number | string');
+    });
+
+    it('should ignore unions inside types', () => {
+      expect(sortUnions(`{x:C|B|A}`)).toEqual(`{x:C|B|A}`);
+      expect(sortUnions(`(x:C|B|A)=>void`)).toEqual(`(x:C|B|A)=>void`);
+      expect(sortUnions(`Partial<C|B|A>`)).toEqual(`Partial<C|B|A>`);
+    });
+
+    it('should sort unions of complex types', () => {
+      expect(sortUnions(`{foo:B|A} | {bar:C|D}`)).toEqual(`{bar:C|D} | {foo:B|A}`);
+    });
+  });
 });
diff --git a/src/test/utils.test.ts b/src/test/utils.test.ts
index 4b1b965..cd1094f 100644
--- a/src/test/utils.test.ts
+++ b/src/test/utils.test.ts
@@ -1,4 +1,4 @@
-import {matchAndExtract} from '../utils';
+import {matchAndExtract, sha256} from '../utils';
 
 describe('utils', () => {
   test('matchAndExtract', () => {
@@ -7,4 +7,10 @@ describe('utils', () => {
     expect(matchAndExtract(pat, 'hello foo bar baz')).toEqual('bar');
     expect(matchAndExtract(pat, 'foo baz')).toEqual(null);
   });
+
+  test('sha256', () => {
+    expect(sha256('')).toMatchInlineSnapshot(
+      `"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"`,
+    );
+  });
 });