Repository: nodejs/node
Pull request: test: add some validation for JSON doc output

We don't have any validation for the JSON output of the docs, errors can easily slip up there. Those tests become more important to gain confidence to switch a different system to generate those.

It's also a way to enforce consistency as demonstrated by the Markdown change in this PR.

<!--
Before submitting a pull request, please read:

- the CONTRIBUTING guide at https://github.com/nodejs/node/blob/HEAD/CONTRIBUTING.md
- the commit message formatting guidelines at
  https://github.com/nodejs/node/blob/HEAD/doc/contributing/pull-requests.md#commit-message-guidelines

For code changes:
1. Include tests for any bug fixes or new features.
2. Update documentation if relevant.
3. Ensure that `make -j4 test` (UNIX), or `vcbuild test` (Windows) passes.

If you believe this PR should be highlighted in the Node.js CHANGELOG
please add the `notable-change` label.

Developer's Certificate of Origin 1.1

By making a contribution to this project, I certify that:

(a) The contribution was created in whole or in part by me and I
    have the right to submit it under the open source license
    indicated in the file; or

(b) The contribution is based upon previous work that, to the best
    of my knowledge, is covered under an appropriate open source
    license and I have the right under that license to submit that
    work with modifications, whether created in whole or in part
    by me, under the same open source license (unless I am
    permitted to submit under a different license), as indicated
    in the file; or

(c) The contribution was provided directly to me by some other
    person who certified (a), (b) or (c) and I have not modified
    it.

(d) I understand and agree that this project and the contribution
    are public and that a record of the contribution (including all
    personal information I submit with it, including my sign-off) is
    maintained indefinitely and may be redistributed consistent with
    this project or the open source license(s) involved.
-->


Changed files:
filename: doc/api/environment_variables.md
patch: @@ -1,5 +1,7 @@
 # Environment Variables
 
+<!--introduced_in=v20.12.0-->
+
 Environment variables are variables associated to the environment the Node.js process runs in.
 
 ## CLI Environment Variables
@@ -20,8 +22,6 @@ Set of utilities for dealing with additional environment variables defined in `.
 
 > Stability: 2 - Stable
 
-<!--introduced_in=v20.12.0-->
-
 ### .env files
 
 `.env` files (also known as dotenv files) are files that define environment variables,

filename: test/doctool/test-doc-api-json.mjs
patch: @@ -0,0 +1,161 @@
+import * as common from '../common/index.mjs';
+
+import assert from 'node:assert';
+import { existsSync } from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+// This tests that `make doc` generates the JSON documentation properly.
+// Note that for this test to pass, `make doc` must be run first.
+
+if (common.isWindows) {
+  common.skip('`make doc` does not run on Windows');
+}
+
+function validateModule(module) {
+  assert.strictEqual(typeof module, 'object');
+  assert.strictEqual(module.type, 'module');
+  assert.ok(module.name);
+  assert.ok(module.textRaw);
+}
+
+function validateMisc(misc) {
+  assert.strictEqual(typeof misc, 'object');
+  assert.strictEqual(misc.type, 'misc');
+  assert.ok(misc.name);
+  assert.strictEqual(typeof misc.name, 'string');
+  assert.ok(misc.textRaw);
+  assert.strictEqual(typeof misc.textRaw, 'string');
+}
+
+let numberOfDeprecatedSections = 0;
+let numberOfRemovedAPIs = 0;
+
+const metaExpectedKeys = new Set([
+  'added',
+  'changes',
+  'deprecated',
+  'napiVersion',
+  'removed',
+]);
+
+function validateMeta(meta) {
+  assert.partialDeepStrictEqual(metaExpectedKeys, new Set(Object.keys(meta)));
+  assert.ok(!Object.hasOwn(meta, 'added') || Array.isArray(meta.added) || typeof meta.added === 'string');
+  if (meta.deprecated) {
+    numberOfDeprecatedSections++;
+    assert.ok(Array.isArray(meta.deprecated) || typeof meta.deprecated === 'string');
+  }
+  if (meta.removed) {
+    numberOfRemovedAPIs++;
+    assert.ok(Array.isArray(meta.removed) || typeof meta.removed === 'string');
+  }
+
+  assert.ok(!Object.hasOwn(meta, 'napiVersion') || Number(meta.napiVersion));
+  assert.ok(Array.isArray(meta.changes));
+}
+
+function findAllKeys(obj, allKeys = new Set()) {
+  for (const [key, value] of Object.entries(obj)) {
+    if (Number.isNaN(Number(key))) allKeys.add(key);
+    if (typeof value === 'object') findAllKeys(value, allKeys);
+
+    if (key === 'miscs') {
+      assert.ok(Array.isArray(value));
+      assert.ok(value.length);
+      value.forEach(validateMisc);
+    } else if (key === 'modules') {
+      assert.ok(Array.isArray(value));
+      assert.ok(value.length);
+      value.forEach(validateModule);
+    } else if (key === 'meta') {
+      validateMeta(value);
+    }
+  }
+  return allKeys;
+}
+
+const allExpectedKeys = new Set([
+  'added',
+  'changes',
+  'classes',
+  'classMethods',
+  'commit',
+  'ctors',
+  'default',
+  'deprecated',
+  'desc',
+  'description',
+  'displayName',
+  'events',
+  'examples',
+  'globals',
+  'introduced_in',
+  'meta',
+  'methods',
+  'miscs',
+  'modules',
+  'name',
+  'napiVersion',
+  'options',
+  'params',
+  'pr-url',
+  'properties',
+  'removed',
+  'return',
+  'shortDesc',
+  'signatures',
+  'source',
+  'stability',
+  'stabilityText',
+  'textRaw',
+  'type',
+  'version',
+]);
+
+for await (const dirent of await fs.opendir(new URL('../../out/doc/api/', import.meta.url))) {
+  if (!dirent.name.endsWith('.md')) continue;
+
+  const jsonPath = path.join(dirent.parentPath, dirent.name.slice(0, -2) + 'json');
+  const expectedSource = `doc/api/${dirent.name}`;
+  if (dirent.name === 'quic.md') {
+    assert.ok(!existsSync(jsonPath)); // QUIC documentation is not public yet
+    continue;
+  }
+
+  console.log('testing', jsonPath, 'based on', expectedSource);
+
+  const fileContent = await fs.readFile(jsonPath, 'utf8');
+  // A proxy to check if the file is human readable is to count if it contains
+  // at least 3 line return.
+  assert.strictEqual(fileContent.split('\n', 3).length, 3);
+
+  const json = JSON.parse(fileContent);
+
+  assert.strictEqual(json.type, 'module');
+  assert.strictEqual(json.source, expectedSource);
+  if (dirent.name !== 'index.md') {
+    assert.ok(json.introduced_in || Object.values(json).at(-1)?.[0].introduced_in);
+  }
+  assert.deepStrictEqual(Object.keys(json), ['type', 'source', ...({
+    'addons.md': ['introduced_in', 'miscs'],
+    'cli.md': ['introduced_in', 'miscs'],
+    'debugger.md': ['introduced_in', 'stability', 'stabilityText', 'miscs'],
+    'deprecations.md': ['introduced_in', 'miscs'],
+    'documentation.md': ['introduced_in', 'miscs'],
+    'errors.md': ['introduced_in', 'classes', 'miscs'],
+    'esm.md': ['introduced_in', 'meta', 'stability', 'stabilityText', 'properties', 'miscs'],
+    'globals.md': ['introduced_in', 'stability', 'stabilityText', 'classes', 'methods', 'miscs'],
+    'index.md': [],
+    'intl.md': ['introduced_in', 'miscs'],
+    'n-api.md': ['introduced_in', 'stability', 'stabilityText', 'miscs'],
+    'packages.md': ['introduced_in', 'meta', 'miscs'],
+    'process.md': ['globals'],
+    'report.md': ['introduced_in', 'stability', 'stabilityText', 'meta', 'miscs'],
+  }[dirent.name] ?? ['modules'])]);
+
+  assert.partialDeepStrictEqual(allExpectedKeys, findAllKeys(json));
+}
+
+assert.strictEqual(numberOfDeprecatedSections, 39); // Increase this number every time a new API is deprecated.
+assert.strictEqual(numberOfRemovedAPIs, 46); // Increase this number every time a section is marked as removed.