Repository: tinylibs/tinybench (ts)

Issue:
Feature request: automatic warmup
It seems a little odd that you must call `.warmup()` even if you've already specified `warmup*` options when initializing the benchmark. I respect the choice to make warmups explicit, but I think there may be an API that accomplishes both.

The idea is to add a `warmup` key to `Options` that can accept `boolean | { time?: number; iterations?: number }`. If this key is specified & truthy, the benchmark will automatically call `.warmup()` when run.

```ts
const a = new Bench({ warmup: true }); // auto-warmup, default config
const b = new Bench({ warmup: { time: 400 }}); // auto-warmup, warmupTime=400
const b = new Bench({ warmup: { iterations: 1000 }}); // auto-warmup, iterations=400
```

This is a non-breaking change since it's a brand new option. Users opt into the "auto warmup" feature by specifying `warmup`. The existing keys `warmupTime` and `warmupIterations` can continue to be supported though you'd presumably throw an informative error if both `warmup.time` and `warmupTime` were specified.

Just a thought. Happy to put in a PR if this seems acceptable. Thanks for the great tool!

Pull request:
feat!: fold warmup step into benchmark run
closes #82

Patch:
diff --git a/README.md b/README.md
index 80f4226..9173680 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,6 @@ bench
     console.log('I am slower');
   });
 
-await bench.warmup(); // make results more reliable
 await bench.run();
 
 console.log(bench.name);
@@ -111,6 +110,11 @@ export interface Options {
    */
   throws?: boolean;
 
+  /**
+   * warmup benchmark @default true
+   */
+  warmup?: boolean;
+
   /**
    * warmup time (milliseconds) @default 100
    */
@@ -136,7 +140,6 @@ export type Hook = (task: Task, mode: 'warmup' | 'run') => void | Promise<void>;
 ```
 
 - `async run()`: run the added tasks that were registered using the `add` method
-- `async warmup()`: warmup the benchmark tasks
 - `reset()`: reset each task and remove its result
 - `add(name: string, fn: Fn, opts?: FnOpts)`: add a benchmark task to the task map
   - `Fn`: `() => unknown | Promise<unknown>`
@@ -392,7 +395,6 @@ It may make your benchmarks slower, check [#42](https://github.com/tinylibs/tiny
 ```ts
 bench.threshold = 10; // The maximum number of concurrent tasks to run. Defaults to Number.POSITIVE_INFINITY.
 bench.concurrency = 'task'; // The concurrency mode to determine how tasks are run.
-await bench.warmup();
 await bench.run();
 ```
 
diff --git a/src/bench.ts b/src/bench.ts
index e126621..a5f45a4 100644
--- a/src/bench.ts
+++ b/src/bench.ts
@@ -52,6 +52,8 @@ export default class Bench extends EventTarget {
 
   throws = false;
 
+  warmup = true;
+
   warmupTime = defaultMinimumWarmupTime;
 
   warmupIterations = defaultMinimumWarmupIterations;
@@ -70,6 +72,7 @@ export default class Bench extends EventTarget {
     super();
     this.name = options.name;
     this.now = options.now ?? this.now;
+    this.warmup = options.warmup ?? this.warmup;
     this.warmupTime = options.warmupTime ?? this.warmupTime;
     this.warmupIterations = options.warmupIterations ?? this.warmupIterations;
     this.time = options.time ?? this.time;
@@ -90,6 +93,25 @@ export default class Bench extends EventTarget {
     }
   }
 
+  /**
+   * warmup the benchmark tasks.
+   */
+  private async warmupTasks(): Promise<void> {
+    this.dispatchEvent(createBenchEvent('warmup'));
+    if (this.concurrency === 'bench') {
+      const limit = pLimit(this.threshold);
+      const promises: Promise<void>[] = [];
+      for (const task of this._tasks.values()) {
+        promises.push(limit(() => task.warmup()));
+      }
+      await Promise.all(promises);
+    } else {
+      for (const task of this._tasks.values()) {
+        await task.warmup();
+      }
+    }
+  }
+
   private async runTask(task: Task): Promise<Task> {
     if (this.signal?.aborted) {
       return task;
@@ -99,9 +121,11 @@ export default class Bench extends EventTarget {
 
   /**
    * run the added tasks that were registered using the {@link add} method.
-   * Note: This method does not do any warmup. Call {@link warmup} for that.
    */
   async run(): Promise<Task[]> {
+    if (this.warmup) {
+      await this.warmupTasks();
+    }
     let values: Task[] = [];
     this.dispatchEvent(createBenchEvent('start'));
     if (this.concurrency === 'bench') {
@@ -120,26 +144,6 @@ export default class Bench extends EventTarget {
     return values;
   }
 
-  /**
-   * warmup the benchmark tasks.
-   * This is not run by default by the {@link run} method.
-   */
-  async warmup(): Promise<void> {
-    this.dispatchEvent(createBenchEvent('warmup'));
-    if (this.concurrency === 'bench') {
-      const limit = pLimit(this.threshold);
-      const promises: Promise<void>[] = [];
-      for (const task of this._tasks.values()) {
-        promises.push(limit(() => task.warmup()));
-      }
-      await Promise.all(promises);
-    } else {
-      for (const task of this._tasks.values()) {
-        await task.warmup();
-      }
-    }
-  }
-
   /**
    * reset each task and remove its result
    */
@@ -196,37 +200,24 @@ export default class Bench extends EventTarget {
   ): (Record<string, string | number> | undefined | null)[] {
     return this.tasks.map((task) => {
       if (task.result) {
-        if (task.result.error) {
-          throw task.result.error;
-        }
-        return (
-          // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
-          convert?.(task) || {
+        return task.result.error
+          ? (convert?.(task) ?? {
+            'Task name': task.name,
+            Error: task.result.error.message,
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            Stack: task.result.error.stack!,
+            Samples: task.result.latency.samples.length,
+          })
+          : (convert?.(task) ?? {
             'Task name': task.name,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            'Throughput average (ops/s)': task.result.error
-              ? 'NaN'
-              : `${task.result.throughput.mean.toFixed(0)} \xb1 ${task.result.throughput.rme.toFixed(2)}%`,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            'Throughput median (ops/s)': task.result.error
-              ? 'NaN'
-              : // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-              `${task.result.throughput.p50!.toFixed(0)}${Number.parseInt(task.result.throughput.mad!.toFixed(0), 10) > 0 ? ` \xb1 ${task.result.throughput.mad!.toFixed(0)}` : ''}`,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            'Latency average (ns)': task.result.error
-              ? 'NaN'
-              : `${(task.result.latency.mean * 1e6).toFixed(2)} \xb1 ${task.result.latency.rme.toFixed(2)}%`,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            'Latency median (ns)': task.result.error
-              ? 'NaN'
-              : // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-              `${(task.result.latency.p50! * 1e6).toFixed(2)}${Number.parseFloat((task.result.latency.mad! * 1e6).toFixed(2)) > 0 ? ` \xb1 ${(task.result.latency.mad! * 1e6).toFixed(2)}` : ''}`,
-            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-            Samples: task.result.error
-              ? 'NaN'
-              : task.result.latency.samples.length,
-          }
-        );
+            'Throughput average (ops/s)': `${task.result.throughput.mean.toFixed(0)} \xb1 ${task.result.throughput.rme.toFixed(2)}%`,
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            'Throughput median (ops/s)': `${task.result.throughput.p50!.toFixed(0)}${Number.parseInt(task.result.throughput.mad!.toFixed(0), 10) > 0 ? ` \xb1 ${task.result.throughput.mad!.toFixed(0)}` : ''}`,
+            'Latency average (ns)': `${(task.result.latency.mean * 1e6).toFixed(2)} \xb1 ${task.result.latency.rme.toFixed(2)}%`,
+            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+            'Latency median (ns)': `${(task.result.latency.p50! * 1e6).toFixed(2)}${Number.parseFloat((task.result.latency.mad! * 1e6).toFixed(2)) > 0 ? ` \xb1 ${(task.result.latency.mad! * 1e6).toFixed(2)}` : ''}`,
+            Samples: task.result.latency.samples.length,
+          });
       }
       return null;
     });
diff --git a/src/task.ts b/src/task.ts
index 118fbae..8f99ccb 100644
--- a/src/task.ts
+++ b/src/task.ts
@@ -130,8 +130,35 @@ export default class Task extends EventTarget {
     return { samples };
   }
 
+  /**
+   * warmup the current task
+   * @internal
+   */
+  async warmup(): Promise<void> {
+    if (this.result?.error) {
+      return;
+    }
+    this.dispatchEvent(createBenchEvent('warmup', this));
+    await this.bench.setup(this, 'warmup');
+    const { error } = (await this.benchmark(
+      this.bench.warmupTime,
+      this.bench.warmupIterations,
+    )) as { error?: Error };
+    await this.bench.teardown(this, 'warmup');
+
+    if (error) {
+      this.setResult({ error });
+      if (this.bench.throws) {
+        throw error;
+      }
+      this.dispatchEvent(createBenchEvent('error', this));
+      this.bench.dispatchEvent(createBenchEvent('error', this));
+    }
+  }
+
   /**
    * run the current task and write the results in `Task.result` object property
+   * @internal
    */
   async run(): Promise<Task> {
     if (this.result?.error) {
@@ -205,29 +232,6 @@ export default class Task extends EventTarget {
     return this;
   }
 
-  /**
-   * warmup the current task
-   */
-  async warmup(): Promise<void> {
-    if (this.result?.error) {
-      return;
-    }
-    this.dispatchEvent(createBenchEvent('warmup', this));
-    await this.bench.setup(this, 'warmup');
-    const { error } = (await this.benchmark(
-      this.bench.warmupTime,
-      this.bench.warmupIterations,
-    )) as { error?: Error };
-    await this.bench.teardown(this, 'warmup');
-
-    if (error) {
-      this.setResult({ error });
-      if (this.bench.throws) {
-        throw error;
-      }
-    }
-  }
-
   addEventListener<K extends TaskEvents>(
     type: K,
     listener: TaskEventsMap[K],
diff --git a/src/types.ts b/src/types.ts
index 865ff85..8c7b4c2 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -338,6 +338,11 @@ export interface Options {
    */
   throws?: boolean;
 
+  /**
+   * warmup benchmark @default true
+   */
+  warmup?: boolean;
+
   /**
    * warmup time (milliseconds) @default 100
    */
diff --git a/src/utils.ts b/src/utils.ts
index 9eadf1a..e621880 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -1,7 +1,7 @@
 import { emptyFunction, tTable } from './constants';
 import type { Fn, Statistics } from './types';
 
-export const nanoToMs = (nano: number) => nano / 1e6;
+const nanoToMs = (nano: number) => nano / 1e6;
 
 const hrtimeBigint = process.hrtime.bigint.bind(process.hrtime);
 export const hrtimeNow = () => nanoToMs(Number(hrtimeBigint()));


Test patch:
diff --git a/test/index.test.ts b/test/index.test.ts
index 7119e37..050767d 100644
--- a/test/index.test.ts
+++ b/test/index.test.ts
@@ -89,8 +89,8 @@ test('events order', async () => {
   const controller = new AbortController();
   const bench = new Bench({
     signal: controller.signal,
-    warmupIterations: 0,
     warmupTime: 0,
+    warmupIterations: 0,
   });
   bench
     // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -165,8 +165,6 @@ test('events order', async () => {
   bench.add('temporary', () => {});
   bench.remove('temporary');
 
-  await bench.warmup();
-
   setTimeout(() => {
     controller.abort();
     // the abort task takes 1000ms (500ms time || 10 iterations => 10 * 1000)
@@ -198,10 +196,7 @@ test('events order', async () => {
 }, 10000);
 
 test('events order at task completion', async () => {
-  const bench = new Bench({
-    warmupIterations: 0,
-    warmupTime: 0,
-  });
+  const bench = new Bench();
 
   bench
     .add('foo', async () => {
@@ -231,8 +226,8 @@ test('events order at task completion', async () => {
   expect(tasks[1]?.name).toBe('bar');
 });
 
-test('error event', async () => {
-  const bench = new Bench({ time: 50 });
+test.each(['warmup', 'run'])('%s error event', async (mode) => {
+  const bench = new Bench({ time: 100, warmup: mode === 'warmup' });
   const err = new Error();
 
   bench.add('error', () => {
@@ -250,8 +245,14 @@ test('error event', async () => {
   expect(taskErr).toBe(err);
 });
 
-test('throws', async () => {
-  const bench = new Bench({ iterations: 1, throws: true });
+test.each(['warmup', 'run'])('%s throws', async (mode) => {
+  const iterations = 1;
+  const bench = new Bench({
+    iterations,
+    throws: true,
+    warmup: mode === 'warmup',
+    warmupIterations: iterations,
+  });
   const err = new Error();
 
   bench.add('error', () => {
@@ -406,7 +407,6 @@ test('setup and teardown', async () => {
   });
   const fooTask = bench.getTask('foo');
 
-  await bench.warmup();
   await bench.run();
 
   expect(setup).toBeCalledWith(fooTask, 'warmup');
@@ -419,8 +419,8 @@ test('task beforeAll, afterAll, beforeEach, afterEach', async () => {
   const iterations = 100;
   const bench = new Bench({
     time: 0,
-    warmupTime: 0,
     iterations,
+    warmupTime: 0,
     warmupIterations: iterations,
   });
 
@@ -449,7 +449,6 @@ test('task beforeAll, afterAll, beforeEach, afterEach', async () => {
     },
   );
 
-  await bench.warmup();
   await bench.run();
 
   expect(beforeAll.mock.calls.length).toBe(2 /* warmup + run */);
diff --git a/test/sequential.test.ts b/test/sequential.test.ts
index 259ff38..6e2e071 100644
--- a/test/sequential.test.ts
+++ b/test/sequential.test.ts
@@ -3,10 +3,15 @@ import { setTimeout } from 'node:timers/promises';
 import { expect, test } from 'vitest';
 import { Bench } from '../src';
 
-test('sequential', async () => {
+test.each(['warmup', 'run'])('%s sequential', async (mode) => {
+  const iterations = 1;
   const sequentialBench = new Bench({
     time: 0,
-    iterations: 100,
+    iterations,
+    throws: true,
+    warmup: mode === 'warmup',
+    warmupTime: 0,
+    warmupIterations: iterations,
   });
 
   const benchTasks: string[] = [];
@@ -14,25 +19,39 @@ test('sequential', async () => {
     .add('sample 1', async () => {
       await setTimeout(randomInt(0, 100));
       benchTasks.push('sample 1');
-      expect(benchTasks).toStrictEqual(['sample 1']);
-      await setTimeout(randomInt(0, 100));
+      if (mode === 'run') {
+        expect(benchTasks).toStrictEqual(['sample 1']);
+      }
     })
     .add('sample 2', async () => {
       await setTimeout(randomInt(0, 100));
       benchTasks.push('sample 2');
-      expect(benchTasks).toStrictEqual(['sample 1', 'sample 2']);
-      await setTimeout(randomInt(0, 100));
+      if (mode === 'run') {
+        expect(benchTasks).toStrictEqual(['sample 1', 'sample 2']);
+      }
     })
     .add('sample 3', async () => {
       await setTimeout(randomInt(0, 100));
       benchTasks.push('sample 3');
-      expect(benchTasks).toStrictEqual(['sample 1', 'sample 2', 'sample 3']);
-      await setTimeout(randomInt(0, 100));
+      if (mode === 'run') {
+        expect(benchTasks).toStrictEqual(['sample 1', 'sample 2', 'sample 3']);
+      }
     });
 
-  await sequentialBench.warmup();
   const tasks = await sequentialBench.run();
 
+  if (mode === 'warmup') {
+    expect(benchTasks).toStrictEqual([
+      'sample 1',
+      'sample 2',
+      'sample 3',
+      'sample 1',
+      'sample 2',
+      'sample 3',
+    ]);
+  } else if (mode === 'run') {
+    expect(benchTasks).toStrictEqual(['sample 1', 'sample 2', 'sample 3']);
+  }
   expect(tasks.length).toBe(3);
   expect(benchTasks.length).toBeGreaterThanOrEqual(tasks.length);
   expect(tasks[0]?.name).toBe('sample 1');
@@ -41,10 +60,14 @@ test('sequential', async () => {
 });
 
 test.each(['warmup', 'run'])('%s bench concurrency', async (mode) => {
+  const iterations = 100;
   const concurrentBench = new Bench({
     time: 0,
-    iterations: 100,
+    iterations,
     throws: true,
+    warmup: mode === 'warmup',
+    warmupTime: 0,
+    warmupIterations: iterations,
   });
   expect(concurrentBench.threshold).toBe(Number.POSITIVE_INFINITY);
   expect(concurrentBench.concurrency).toBeNull();
@@ -68,15 +91,9 @@ test.each(['warmup', 'run'])('%s bench concurrency', async (mode) => {
       shouldNotBeDefinedFirst2 = true;
     });
 
-  if (mode === 'warmup') {
-    // not awaited on purpose
-    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    concurrentBench.warmup();
-  } else if (mode === 'run') {
-    // not awaited on purpose
-    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-    concurrentBench.run();
-  }
+  // not awaited on purpose
+  // eslint-disable-next-line @typescript-eslint/no-floating-promises
+  concurrentBench.run();
 
   await setTimeout(0);
   expect(shouldBeDefined1).toBeDefined();
@@ -92,8 +109,10 @@ test.each(['warmup', 'run'])('%s task concurrency', async (mode) => {
   const iterations = 10;
   const concurrentBench = new Bench({
     time: 0,
-    warmupTime: 0,
     iterations,
+    throws: true,
+    warmup: mode === 'warmup',
+    warmupTime: 0,
     warmupIterations: iterations,
   });
   expect(concurrentBench.threshold).toBe(Number.POSITIVE_INFINITY);
@@ -104,22 +123,20 @@ test.each(['warmup', 'run'])('%s task concurrency', async (mode) => {
 
   concurrentBench.add(taskName, async () => {
     runs++;
-    await setTimeout(10);
-    // all task function should be here after 10ms
-    expect(runs).toEqual(iterations);
-    await setTimeout(10);
+    if (concurrentBench.concurrency === 'task') {
+      await setTimeout(10);
+      // all task function should be here after 10ms
+      expect([iterations, 2 * iterations]).toContain(runs);
+    }
   });
 
-  if (mode === 'warmup') {
-    await concurrentBench.warmup();
-  } else if (mode === 'run') {
-    await concurrentBench.run();
-    for (const result of concurrentBench.results) {
-      expect(result?.error).toMatchObject(/AssertionError/);
-    }
+  await concurrentBench.run();
+
+  for (const result of concurrentBench.results) {
+    expect(result?.error).toMatchObject(/AssertionError/);
   }
-  expect(concurrentBench.getTask(taskName)?.runs).toEqual(0);
-  expect(runs).toEqual(1);
+  expect(concurrentBench.getTask(taskName)?.runs).toEqual(iterations);
+  expect(runs).toEqual(mode === 'run' ? iterations : 2 * iterations);
   concurrentBench.reset();
   runs = 0;
   expect(concurrentBench.threshold).toBe(Number.POSITIVE_INFINITY);
@@ -129,14 +146,11 @@ test.each(['warmup', 'run'])('%s task concurrency', async (mode) => {
   expect(concurrentBench.threshold).toBe(Number.POSITIVE_INFINITY);
   expect(concurrentBench.concurrency).toBe('task');
 
-  if (mode === 'warmup') {
-    await concurrentBench.warmup();
-  } else if (mode === 'run') {
-    await concurrentBench.run();
-    for (const result of concurrentBench.results) {
-      expect(result?.error).toBeUndefined();
-    }
-    expect(concurrentBench.getTask(taskName)?.runs).toEqual(10);
+  await concurrentBench.run();
+
+  for (const result of concurrentBench.results) {
+    expect(result?.error).toBeUndefined();
   }
-  expect(runs).toEqual(10);
+  expect(concurrentBench.getTask(taskName)?.runs).toEqual(iterations);
+  expect(runs).toEqual(mode === 'run' ? iterations : 2 * iterations);
 });