Repository: avajs/ava
Pull request #2962 (2022-02-06 18:07:13)

Warn when unsupported ava.config.json files are encountered
See https://github.com/avajs/ava/issues/2957#issuecomment-1030884141

novemberborn: Thanks for this @razor-x. `ava.config.json` files have actually never worked, it's not a v4 thing.

I'm thinking we shouldn't print to the console in `load-config` itself. Perhaps we could return an array of existing `ava.config.json` paths that were encountered while loading a configuration file, and then do the logging from `lib/cli.js`?
novemberborn: Returning it makes it easier to print a nice list of encountered files, that's harder to do with a callback.
razor-x: I looked at updating the return value, but that seemed kinda stressful as it's a breaking change for that function and would require updating its usage in all tests and other files. I ended up opting for the callback instead since it's a backwards compatible change. Hopefully this is ok with you. It still produces a nice list since it calls the function once with an array.

I ran a quick local test on the cli to check the warning. This is what it looks like.

<img width="544" alt="2022-02-13_21-22" src="https://user-images.githubusercontent.com/721372/153805005-32e368c6-c78c-43b1-a0b8-33da82334c6a.png">

novemberborn: Cheers @razor-x. I haven't had a chance to look at this yet though.
razor-x: No worries. I fixed the tests.

Patch:
diff --git a/docs/05-command-line.md b/docs/05-command-line.md
index 49743ff3..ea1f25da 100644
--- a/docs/05-command-line.md
+++ b/docs/05-command-line.md
@@ -12,36 +12,38 @@ Commands:
   ava debug [<pattern>...]  Activate Node.js inspector and run a single test
                             file
   ava reset-cache           Delete any temporary files and state kept by AVA,
-	                          then exit
+                            then exit
 
 Positionals:
   pattern  Select which test files to run. Leave empty if you want AVA to run
            all test files as per your configuration. Accepts glob patterns,
-           directories that (recursively) contain test files, and file paths.
-           Add a colon and specify line numbers of specific tests
-           to run                                                       [string]
+           directories that (recursively) contain test files, and file paths
+           optionally suffixed with a colon and comma-separated numbers and/or
+           ranges identifying the 1-based line(s) of specific tests to run
+                                                                        [string]
+
 Options:
-  --version               Show version number                          [boolean]
-  --color                 Force color output                           [boolean]
-  --config                Specific JavaScript file for AVA to read its config
-                          from, instead of using package.json or ava.config.*
-                          files
-  --help                  Show help                                    [boolean]
-  --concurrency, -c       Max number of test files running at the same time
-                          (default: CPU cores)                          [number]
-  --no-worker-threads     Don't use worker threads                     [boolean]
-  --fail-fast             Stop after first test failure                [boolean]
-  --match, -m             Only run tests with matching title (can be repeated)
+      --version            Show version number                         [boolean]
+      --color              Force color output                          [boolean]
+      --config             Specific JavaScript file for AVA to read its config
+                           from, instead of using package.json or ava.config.*
+                           files
+      --help               Show help                                   [boolean]
+  -c, --concurrency        Max number of test files running at the same time
+                           (default: CPU cores)                         [number]
+      --fail-fast          Stop after first test failure               [boolean]
+  -m, --match              Only run tests with matching title (can be repeated)
                                                                         [string]
-  --node-arguments        Additional Node.js arguments for launching worker
-                          processes (specify as a single string)        [string]
-  --serial, -s            Run tests serially                           [boolean]
-  --tap, -t               Generate TAP output                          [boolean]
-  --timeout, -T           Set global timeout (milliseconds or human-readable,
-                          e.g. 10s, 2m)                                 [string]
-  --update-snapshots, -u  Update snapshots                             [boolean]
-  --verbose, -v           Enable verbose output (default)              [boolean]
-  --watch, -w             Re-run tests when files change               [boolean]
+      --no-worker-threads  Don't use worker threads                    [boolean]
+      --node-arguments     Additional Node.js arguments for launching worker
+                           processes (specify as a single string)       [string]
+  -s, --serial             Run tests serially                          [boolean]
+  -t, --tap                Generate TAP output                         [boolean]
+  -T, --timeout            Set global timeout (milliseconds or human-readable,
+                           e.g. 10s, 2m)                                [string]
+  -u, --update-snapshots   Update snapshots                            [boolean]
+  -v, --verbose            Enable verbose output (default)             [boolean]
+  -w, --watch              Re-run tests when files change              [boolean]
 
 Examples:
   ava
diff --git a/docs/06-configuration.md b/docs/06-configuration.md
index bd6545af..ccf081d5 100644
--- a/docs/06-configuration.md
+++ b/docs/06-configuration.md
@@ -58,6 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con
 - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation)
 - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options.
 - `nodeArguments`: Configure Node.js arguments used to launch worker processes.
+- `sortTestFiles`: A comparator function to sort test files with. Available only when using a `ava.config.*` file. See an example use case [here](recipes/splitting-tests-ci.md).
 
 Note that providing files on the CLI overrides the `files` option.
 
diff --git a/lib/api.js b/lib/api.js
index bf0cb249..1ebe27ce 100644
--- a/lib/api.js
+++ b/lib/api.js
@@ -177,13 +177,19 @@ export default class Api extends Emittery {
 				const fileCount = selectedFiles.length;
 
 				// The files must be in the same order across all runs, so sort them.
-				selectedFiles = selectedFiles.sort((a, b) => a.localeCompare(b, [], {numeric: true}));
+				const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true});
+				selectedFiles = selectedFiles.sort(this.options.sortTestFiles || defaultComparator);
 				selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns);
 
 				const currentFileCount = selectedFiles.length;
 
 				runStatus = new RunStatus(fileCount, {currentFileCount, currentIndex, totalRuns}, selectionInsights);
 			} else {
+				// If a custom sorter was configured, use it.
+				if (this.options.sortTestFiles) {
+					selectedFiles = selectedFiles.sort(this.options.sortTestFiles);
+				}
+
 				runStatus = new RunStatus(selectedFiles.length, null, selectionInsights);
 			}
 
@@ -261,8 +267,8 @@ export default class Api extends Emittery {
 				}
 
 				const lineNumbers = getApplicableLineNumbers(globs.normalizeFileForMatching(apiOptions.projectDir, file), filter);
-				// Removing `providers` field because they cannot be transfered to the worker threads.
-				const {providers, ...forkOptions} = apiOptions;
+				// Removing `providers` and `sortTestFiles` fields because they cannot be transferred to the worker threads.
+				const {providers, sortTestFiles, ...forkOptions} = apiOptions;
 				const options = {
 					...forkOptions,
 					providerStates,
diff --git a/lib/cli.js b/lib/cli.js
index f1597950..48276685 100644
--- a/lib/cli.js
+++ b/lib/cli.js
@@ -104,7 +104,14 @@ export default async function loadCli() { // eslint-disable-line complexity
 	let confError;
 	try {
 		const {argv: {config: configFile}} = yargs(hideBin(process.argv)).help(false).version(false);
-		conf = await loadConfig({configFile});
+		const loaded = await loadConfig({configFile});
+		if (loaded.unsupportedFiles.length > 0) {
+			console.log(chalk.magenta(
+				`  ${figures.warning} AVA does not support JSON config, ignoring:\n\n    ${loaded.unsupportedFiles.join('\n    ')}`,
+			));
+		}
+
+		conf = loaded.config;
 		if (conf.configFile && path.basename(conf.configFile) !== path.relative(conf.projectDir, conf.configFile)) {
 			console.log(chalk.magenta(`  ${figures.warning} Using configuration from ${conf.configFile}`));
 		}
@@ -163,7 +170,7 @@ export default async function loadCli() { // eslint-disable-line complexity
 		})
 		.command('* [<pattern>...]', 'Run tests', yargs => yargs.options(FLAGS).positional('pattern', {
 			array: true,
-			describe: 'Select which test files to run. Leave empty if you want AVA to run all test files as per your configuration. Accepts glob patterns, directories that (recursively) contain test files, and file paths. Add a colon and specify line numbers of specific tests to run',
+			describe: 'Select which test files to run. Leave empty if you want AVA to run all test files as per your configuration. Accepts glob patterns, directories that (recursively) contain test files, and file paths optionally suffixed with a colon and comma-separated numbers and/or ranges identifying the 1-based line(s) of specific tests to run',
 			type: 'string',
 		}), argv => {
 			if (activeInspector) {
@@ -190,7 +197,7 @@ export default async function loadCli() { // eslint-disable-line complexity
 				},
 			}).positional('pattern', {
 				demand: true,
-				describe: 'Glob patterns to select a single test file to debug. Add a colon and specify line numbers of specific tests to run',
+				describe: 'Glob pattern to select a single test file to debug, optionally suffixed with a colon and comma-separated numbers and/or ranges identifying the 1-based line(s) of specific tests to run',
 				type: 'string',
 			}),
 			argv => {
@@ -321,6 +328,10 @@ export default async function loadCli() { // eslint-disable-line complexity
 		exit('’sources’ has been removed. Use ’ignoredByWatcher’ to provide glob patterns of files that the watcher should ignore.');
 	}
 
+	if (Reflect.has(conf, 'sortTestFiles') && typeof conf.sortTestFiles !== 'function') {
+		exit('’sortTestFiles’ must be a comparator function.');
+	}
+
 	let projectPackageObject;
 	try {
 		projectPackageObject = JSON.parse(fs.readFileSync(path.resolve(projectDir, 'package.json')));
@@ -413,6 +424,7 @@ export default async function loadCli() { // eslint-disable-line complexity
 		moduleTypes,
 		nodeArguments,
 		parallelRuns,
+		sortTestFiles: conf.sortTestFiles,
 		projectDir,
 		providers,
 		ranFromCli: true,
diff --git a/lib/eslint-plugin-helper-worker.js b/lib/eslint-plugin-helper-worker.js
index e53bca40..fb7ce605 100644
--- a/lib/eslint-plugin-helper-worker.js
+++ b/lib/eslint-plugin-helper-worker.js
@@ -41,7 +41,7 @@ const buildGlobs = ({conf, providers, projectDir, overrideExtensions, overrideFi
 
 const resolveGlobs = async (projectDir, overrideExtensions, overrideFiles) => {
 	if (!configCache.has(projectDir)) {
-		configCache.set(projectDir, loadConfig({resolveFrom: projectDir}).then(async conf => {
+		configCache.set(projectDir, loadConfig({resolveFrom: projectDir}).then(async ({config: conf}) => {
 			const providers = await collectProviders({conf, projectDir});
 			return {conf, providers};
 		}));
diff --git a/lib/load-config.js b/lib/load-config.js
index e02809ed..e3c70fd7 100644
--- a/lib/load-config.js
+++ b/lib/load-config.js
@@ -20,14 +20,15 @@ const importConfig = async ({configFile, fileForErrorMessage}) => {
 };
 
 const loadConfigFile = async ({projectDir, configFile}) => {
-	if (!fs.existsSync(configFile)) {
-		return null;
-	}
-
 	const fileForErrorMessage = path.relative(projectDir, configFile);
 	try {
+		await fs.promises.access(configFile);
 		return {config: await importConfig({configFile, fileForErrorMessage}), configFile, fileForErrorMessage};
 	} catch (error) {
+		if (error.code === 'ENOENT') {
+			return null;
+		}
+
 		throw Object.assign(new Error(`Error loading ${fileForErrorMessage}: ${error.message}`), {parent: error});
 	}
 };
@@ -63,6 +64,20 @@ async function findRepoRoot(fromDir) {
 	return root;
 }
 
+async function checkJsonFile(searchDir) {
+	const file = path.join(searchDir, 'ava.config.json');
+	try {
+		await fs.promises.access(file);
+		return file;
+	} catch (error) {
+		if (error.code === 'ENOENT') {
+			return null;
+		}
+
+		throw error;
+	}
+}
+
 export async function loadConfig({configFile, resolveFrom = process.cwd(), defaults = {}} = {}) {
 	let packageConf = await packageConfig('ava', {cwd: resolveFrom});
 	const filepath = packageJsonPath(packageConf);
@@ -74,6 +89,7 @@ export async function loadConfig({configFile, resolveFrom = process.cwd(), defau
 	const allowConflictWithPackageJson = Boolean(configFile);
 	configFile = resolveConfigFile(configFile);
 
+	const unsupportedFiles = [];
 	let fileConf = NO_SUCH_FILE;
 	let fileForErrorMessage;
 	let conflicting = [];
@@ -86,12 +102,17 @@ export async function loadConfig({configFile, resolveFrom = process.cwd(), defau
 		let searchDir = projectDir;
 		const stopAt = path.dirname(repoRoot);
 		do {
-			const results = await Promise.all([ // eslint-disable-line no-await-in-loop
+			const [jsonFile, ...results] = await Promise.all([ // eslint-disable-line no-await-in-loop
+				checkJsonFile(searchDir),
 				loadConfigFile({projectDir, configFile: path.join(searchDir, 'ava.config.js')}),
 				loadConfigFile({projectDir, configFile: path.join(searchDir, 'ava.config.cjs')}),
 				loadConfigFile({projectDir, configFile: path.join(searchDir, 'ava.config.mjs')}),
 			]);
 
+			if (jsonFile !== null) {
+				unsupportedFiles.push(jsonFile);
+			}
+
 			[{config: fileConf, fileForErrorMessage, configFile} = {config: NO_SUCH_FILE, fileForErrorMessage: undefined}, ...conflicting] = results.filter(result => result !== null);
 
 			searchDir = path.dirname(searchDir);
@@ -139,5 +160,5 @@ export async function loadConfig({configFile, resolveFrom = process.cwd(), defau
 		}
 	}
 
-	return config;
+	return {config, unsupportedFiles};
 }
diff --git a/readme.md b/readme.md
index ab865265..78b4a9cd 100644
--- a/readme.md
+++ b/readme.md
@@ -139,15 +139,16 @@ We have a growing list of [common pitfalls](docs/08-common-pitfalls.md) you may
 
 ### Recipes
 
-- [Shared workers](docs/recipes/shared-workers.md)
 - [Test setup](docs/recipes/test-setup.md)
-- [Code coverage](docs/recipes/code-coverage.md)
+- [TypeScript](docs/recipes/typescript.md)
+- [Shared workers](docs/recipes/shared-workers.md)
 - [Watch mode](docs/recipes/watch-mode.md)
-- [Endpoint testing](docs/recipes/endpoint-testing.md)
 - [When to use `t.plan()`](docs/recipes/when-to-use-plan.md)
-- [Browser testing](docs/recipes/browser-testing.md)
-- [TypeScript](docs/recipes/typescript.md)
 - [Passing arguments to your test files](docs/recipes/passing-arguments-to-your-test-files.md)
+- [Splitting tests in CI](docs/recipes/splitting-tests-ci.md)
+- [Code coverage](docs/recipes/code-coverage.md)
+- [Endpoint testing](docs/recipes/endpoint-testing.md)
+- [Browser testing](docs/recipes/browser-testing.md)
 - [Testing Vue.js components](docs/recipes/vue.md)
 - [Debugging tests with Chrome DevTools](docs/recipes/debugging-with-chrome-devtools.md)
 - [Debugging tests with VSCode](docs/recipes/debugging-with-vscode.md)


Test patch:
diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md
new file mode 100644
index 00000000..95849d3d
--- /dev/null
+++ b/docs/recipes/splitting-tests-ci.md
@@ -0,0 +1,22 @@
+# Splitting tests in CI
+
+AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. Each CI machine is assigned a chunk (subset) of the tests, and then each chunk is run in parallel.
+
+To better distribute the tests across the machines, you can configure a custom comparator function:
+
+**`ava.config.js`:**
+
+```js
+import fs from 'node:fs';
+
+// Assuming 'test-data.json' structure is:
+// {
+// 	'tests/test1.js': { order: 1 },
+// 	'tests/test2.js': { order: 0 }
+// }
+const testData = JSON.parse(fs.readFileSync('test-data.json', 'utf8'));
+
+export default {
+	sortTestFiles: (file1, file2) => testData[file1].order - testData[file2].order,
+};
+```
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs
new file mode 100644
index 00000000..5b1c0566
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs
new file mode 100644
index 00000000..5b1c0566
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs
new file mode 100644
index 00000000..5b1c0566
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs
new file mode 100644
index 00000000..ccb99916
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs
new file mode 100644
index 00000000..ccb99916
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs
new file mode 100644
index 00000000..ccb99916
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs
new file mode 100644
index 00000000..a40e62fb
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs
new file mode 100644
index 00000000..a40e62fb
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs
new file mode 100644
index 00000000..a40e62fb
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js
new file mode 100644
index 00000000..935e14cf
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js
@@ -0,0 +1,5 @@
+export default {
+	files: ['*.cjs'],
+	// Descending order
+	sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}),
+};
diff --git a/test-tap/fixture/parallel-runs/custom-comparator/package.json b/test-tap/fixture/parallel-runs/custom-comparator/package.json
new file mode 100644
index 00000000..bedb411a
--- /dev/null
+++ b/test-tap/fixture/parallel-runs/custom-comparator/package.json
@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}
diff --git a/test-tap/fixture/sort-tests/0.cjs b/test-tap/fixture/sort-tests/0.cjs
new file mode 100644
index 00000000..1af77b59
--- /dev/null
+++ b/test-tap/fixture/sort-tests/0.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run third', t => {
+	t.pass();
+});
diff --git a/test-tap/fixture/sort-tests/1.cjs b/test-tap/fixture/sort-tests/1.cjs
new file mode 100644
index 00000000..e1e0ff47
--- /dev/null
+++ b/test-tap/fixture/sort-tests/1.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run second', t => {
+	t.pass();
+});
diff --git a/test-tap/fixture/sort-tests/2.cjs b/test-tap/fixture/sort-tests/2.cjs
new file mode 100644
index 00000000..9ad7df09
--- /dev/null
+++ b/test-tap/fixture/sort-tests/2.cjs
@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run first', t => {
+	t.pass();
+});
diff --git a/test-tap/fixture/sort-tests/ava.config.js b/test-tap/fixture/sort-tests/ava.config.js
new file mode 100644
index 00000000..58d09828
--- /dev/null
+++ b/test-tap/fixture/sort-tests/ava.config.js
@@ -0,0 +1,7 @@
+export default {
+	files: ['*.cjs'],
+	// Descending order
+	sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}),
+	concurrency: 1,
+	verbose: true,
+};
diff --git a/test-tap/fixture/sort-tests/package.json b/test-tap/fixture/sort-tests/package.json
new file mode 100644
index 00000000..bedb411a
--- /dev/null
+++ b/test-tap/fixture/sort-tests/package.json
@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}
diff --git a/test-tap/integration/assorted.js b/test-tap/integration/assorted.js
index e866824c..db292e8a 100644
--- a/test-tap/integration/assorted.js
+++ b/test-tap/integration/assorted.js
@@ -152,3 +152,11 @@ test('load .js test files as ESM modules', t => {
 		t.end();
 	});
 });
+
+test('uses sortTestFiles to sort test files', t => {
+	execCli([], {dirname: 'fixture/sort-tests'}, (error, stdout) => {
+		t.error(error);
+		t.match(stdout, /should run first[\s\S]+?should run second[\s\S]+?should run third/);
+		t.end();
+	});
+});
diff --git a/test-tap/integration/parallel-runs.js b/test-tap/integration/parallel-runs.js
index fe30e38c..47f4c078 100644
--- a/test-tap/integration/parallel-runs.js
+++ b/test-tap/integration/parallel-runs.js
@@ -43,3 +43,17 @@ test('fail when there are no files', t => {
 		}, error => t.ok(error));
 	}
 });
+
+test('correctly applies custom comparator', t => {
+	t.plan(3);
+	for (let i = 0; i < 3; i++) {
+		execCli([], {
+			dirname: 'fixture/parallel-runs/custom-comparator',
+			env: {
+				AVA_FORCE_CI: 'ci',
+				CI_NODE_INDEX: String(i),
+				CI_NODE_TOTAL: '3',
+			},
+		}, error => t.error(error));
+	}
+});
diff --git a/test/config/fixtures/unsupported-configs/ava.config.js b/test/config/fixtures/unsupported-configs/ava.config.js
new file mode 100644
index 00000000..4a9fb4c2
--- /dev/null
+++ b/test/config/fixtures/unsupported-configs/ava.config.js
@@ -0,0 +1,3 @@
+export default {
+	failFast: true,
+};
diff --git a/test/config/fixtures/unsupported-configs/ava.config.json b/test/config/fixtures/unsupported-configs/ava.config.json
new file mode 100644
index 00000000..0967ef42
--- /dev/null
+++ b/test/config/fixtures/unsupported-configs/ava.config.json
@@ -0,0 +1,1 @@
+{}
diff --git a/test/config/fixtures/unsupported-configs/package.json b/test/config/fixtures/unsupported-configs/package.json
new file mode 100644
index 00000000..bedb411a
--- /dev/null
+++ b/test/config/fixtures/unsupported-configs/package.json
@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}
diff --git a/test/config/loader.js b/test/config/loader.js
index 1be2d0ec..8f4c5ae5 100644
--- a/test/config/loader.js
+++ b/test/config/loader.js
@@ -10,13 +10,15 @@ const FIXTURE_ROOT = fileURLToPath(new URL('../../test-tap/fixture/load-config',
 
 const resolve = relpath => path.resolve(FIXTURE_ROOT, relpath);
 
-const loadFromSetup = setup => {
+const loadFromSetup = async setup => {
 	if (typeof setup === 'string') {
-		return loadConfig();
+		const loaded = await loadConfig();
+		return loaded.config;
 	}
 
 	const {configFile, defaults, resolveFrom} = setup;
-	return loadConfig({configFile, defaults, resolveFrom});
+	const loaded = await loadConfig({configFile, defaults, resolveFrom});
+	return loaded.config;
 };
 
 const ok = setup => async (t, assert = tt => tt.pass()) => {
diff --git a/test/config/next-gen.js b/test/config/next-gen.js
index 5e62efd2..2bcf4b87 100644
--- a/test/config/next-gen.js
+++ b/test/config/next-gen.js
@@ -10,23 +10,31 @@ const FIXTURE_ROOT = fileURLToPath(new URL('fixtures', import.meta.url));
 
 const resolve = relpath => path.resolve(FIXTURE_ROOT, relpath);
 
-const loadFromSetup = setup => {
+const loadFromSetup = async (setup, t, assertUnsupportedFiles = (tt, files) => tt.is(files.length, 0)) => {
 	if (typeof setup === 'string') {
-		return loadConfig();
+		const loaded = await loadConfig();
+		return loaded.config;
 	}
 
-	const {configFile, defaults, resolveFrom} = setup;
-	return loadConfig({configFile, defaults, resolveFrom});
+	const {
+		configFile,
+		defaults,
+		resolveFrom,
+	} = setup;
+
+	const loaded = await loadConfig({configFile, defaults, resolveFrom});
+	assertUnsupportedFiles(t, loaded.unsupportedFiles);
+	return loaded.config;
 };
 
-const ok = setup => async (t, assert = tt => tt.pass()) => {
+const ok = setup => async (t, assert = tt => tt.pass(), assertUnsupportedFiles = undefined) => {
 	const fixture = typeof setup === 'string' ? setup : setup.fixture;
 
 	const stub = sinon.stub(process, 'cwd');
 	t.teardown(() => stub.restore());
 	stub.returns(resolve(fixture));
 
-	const conf = loadFromSetup(setup);
+	const conf = loadFromSetup(setup, t, assertUnsupportedFiles);
 	await t.notThrowsAsync(conf);
 	const result = await t.try(assert, await conf, setup);
 	result.commit();
@@ -39,7 +47,7 @@ const notOk = setup => async (t, assert = (tt, error) => tt.snapshot(error.messa
 	t.teardown(() => stub.restore());
 	stub.returns(resolve(fixture));
 
-	const conf = loadFromSetup(setup);
+	const conf = loadFromSetup(setup, t);
 	const error = await t.throwsAsync(conf);
 	const result = await t.try(assert, error, setup);
 	result.commit();
@@ -67,6 +75,19 @@ test.serial('loads .js config as ESM', ok('js-as-esm'), (t, conf) => {
 	t.true(conf.failFast);
 });
 
+test.serial('finds unsupported configs',
+	ok({
+		fixture: 'unsupported-configs',
+	}),
+	(t, conf) => {
+		t.true(conf.failFast);
+	},
+	(t, unsupportedFiles) => {
+		t.is(unsupportedFiles.length, 1);
+		t.regex(unsupportedFiles[0], /ava\.config\.json/);
+	},
+);
+
 test.serial('handles errors when loading .js config as ESM', notOk({
 	fixture: 'js-as-esm',
 	configFile: 'error.js',
diff --git a/test/node_modules/.bin/test-ava b/test/node_modules/.bin/test-ava
deleted file mode 120000
index bd4f1e8f..00000000
--- a/test/node_modules/.bin/test-ava
+++ /dev/null
@@ -1,1 +0,0 @@
-/Users/jsamines/dev/personal/ava/node_modules/.bin/test-ava
\ No newline at end of file