Repository: promptfoo/promptfoo
Pull request #78 (2023-07-20 17:03:40)

Add `python` assertion type that runs python code and returns the output
This makes life easier for users who are more comfortable in the python ecosystem.  

cloudflare-pages[bot]: ## Deploying with &nbsp;<a href="https://pages.dev"><img alt="Cloudflare Pages" src="https://user-images.githubusercontent.com/23264/106598434-9e719e00-654f-11eb-9e59-6167043cfa01.png" width="16"></a> &nbsp;Cloudflare Pages

<table><tr><td><strong>Latest commit:</strong> </td><td>
<code>7866329</code>
</td></tr>
<tr><td><strong>Status:</strong></td><td>&nbsp;✅&nbsp; Deploy successful!</td></tr>
<tr><td><strong>Preview URL:</strong></td><td>
<a href='https://f215a777.promptfoo-2.pages.dev'>https://f215a777.promptfoo-2.pages.dev</a>
</td></tr>
<tr><td><strong>Branch Preview URL:</strong></td><td>
<a href='https://python-assertion.promptfoo-2.pages.dev'>https://python-assertion.promptfoo-2.pages.dev</a>
</td></tr>
</table>

[View logs](https://dash.cloudflare.com/?to=/:account/pages/view/promptfoo-2/f215a777-4587-485d-b52a-597bb9e27655)


Patch:
diff --git a/README.md b/README.md
index 66b9fc092..9c280c00c 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
 
 [![npm](https://img.shields.io/npm/v/promptfoo)](https://npmjs.com/package/promptfoo)
 [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/typpo/promptfoo/main.yml)](https://github.com/typpo/promptfoo/actions/workflows/main.yml)
+![MIT license](https://img.shields.io/github/license/typpo/promptfoo)
 
 `promptfoo` is a tool for testing and evaluating LLM prompt quality.
 
@@ -102,6 +103,7 @@ See [Test assertions](https://promptfoo.dev/docs/configuration/expected-outputs)
 | `is-json`       | output is valid json                                                      |
 | `contains-json` | output contains valid json                                                |
 | `javascript`    | provided Javascript function validates the output                         |
+| `python`        | provided Python function validates the output                             |
 | `webhook`       | provided webhook returns `{pass: true}`                                   |
 | `similar`       | embeddings and cosine similarity are above a threshold                    |
 | `llm-rubric`    | LLM output matches a given rubric, using a Language Model to grade output |
diff --git a/examples/langchain-python/requirements.txt b/examples/langchain-python/requirements.txt
index 6c0103ef0..66619acae 100644
--- a/examples/langchain-python/requirements.txt
+++ b/examples/langchain-python/requirements.txt
@@ -1,4 +1,4 @@
-aiohttp==3.8.4
+aiohttp==3.8.5
 aiosignal==1.3.1
 async-timeout==4.0.2
 attrs==23.1.0
diff --git a/examples/llama-gpt-comparison/README.md b/examples/llama-gpt-comparison/README.md
index 6fd2d593f..9fa898324 100644
--- a/examples/llama-gpt-comparison/README.md
+++ b/examples/llama-gpt-comparison/README.md
@@ -3,6 +3,7 @@ To get started, set your OPENAI_API_KEY and REPLICATE_API_TOKEN environment vari
 Next, change a few of the prompts in prompts.txt and edit promptfooconfig.yaml.
 
 Then run:
+
 ```
 promptfoo eval
 ```
diff --git a/examples/llama-gpt-comparison/promptfooconfig.yaml b/examples/llama-gpt-comparison/promptfooconfig.yaml
index 4aa3929b7..12535275e 100644
--- a/examples/llama-gpt-comparison/promptfooconfig.yaml
+++ b/examples/llama-gpt-comparison/promptfooconfig.yaml
@@ -16,7 +16,7 @@ providers:
   - replicate:replicate/llama70b-v2-chat:e951f18578850b652510200860fc4ea62b3b16fac280f83ff32282f87bbd2e48:
       prompts: completion_prompt
       config:
-        temperature: 0.01  # minimum temperature
+        temperature: 0.01 # minimum temperature
         max_length: 128
 
 tests:
diff --git a/package-lock.json b/package-lock.json
index d9d4fd88a..9c5f6982a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "promptfoo",
-  "version": "0.17.6",
+  "version": "0.17.9",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "promptfoo",
-      "version": "0.17.6",
+      "version": "0.17.9",
       "license": "MIT",
       "dependencies": {
         "@anthropic-ai/sdk": "^0.5.2",
@@ -62,7 +62,7 @@
         "typescript": "^5.0.4"
       },
       "engines": {
-        "node": ">=12"
+        "node": ">=16"
       }
     },
     "node_modules/@ampproject/remapping": {
diff --git a/package.json b/package.json
index 9425aed92..419e0a227 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
   "name": "promptfoo",
   "description": "LLM eval & testing toolkit",
   "author": "Ian Webster",
-  "version": "0.17.6",
+  "version": "0.17.9",
   "license": "MIT",
   "type": "commonjs",
   "main": "dist/src/index.js",
@@ -19,7 +19,7 @@
     "src"
   ],
   "engines": {
-    "node": ">=12"
+    "node": ">=16"
   },
   "bin": {
     "promptfoo": "dist/src/main.js"
diff --git a/src/assertions.ts b/src/assertions.ts
index 9cebfa042..cb5c78b4f 100644
--- a/src/assertions.ts
+++ b/src/assertions.ts
@@ -234,12 +234,13 @@ export async function runAssertion(
     };
   }
 
+  const context = {
+    vars: test.vars || {},
+  };
+
   if (baseType === 'javascript') {
     try {
       const customFunction = new Function('output', 'context', `return ${assertion.value}`);
-      const context = {
-        vars: test.vars || {},
-      };
       const result = customFunction(output, context) as any;
       if (typeof result === 'boolean') {
         pass = result !== inverse;
@@ -268,6 +269,44 @@ ${assertion.value}`,
     };
   }
 
+  if (baseType === 'python') {
+    try {
+      const { execSync } = require('child_process');
+      const escapedOutput = output.replace(/'/g, "\\'").replace(/"/g, '\\"');;
+      const escapedContext = JSON.stringify(context).replace(/'/g, "\\'").replace(/"/g, '\\"');
+      const result = execSync(`python -c "import json; import math; import os; import sys; import re; import datetime; import random; import collections; output='${escapedOutput}'; context='${escapedContext}'; print(json.dumps(${assertion.value}))"`).toString().trim();
+      if (result === 'true') {
+        pass = true;
+        score = 1.0;
+      } else if (result === 'false') {
+        pass = false;
+        score = 0.0;
+      } else if (result.startsWith('{')) {
+        return JSON.parse(result);
+      } else {
+        pass = true;
+        score = parseFloat(result);
+        if (isNaN(score)) {
+          throw new Error('Python code must return a boolean, number, or {pass, score, reason} object');
+        }
+      }
+    } catch (err) {
+      return {
+        pass: false,
+        score: 0,
+        reason: `Python code execution failed: ${(err as Error).message}`,
+      };
+    }
+    return {
+      pass,
+      score,
+      reason: pass
+        ? 'Assertion passed'
+        : `Python code returned ${pass ? 'true' : 'false'}
+${assertion.value}`,
+    };
+  }
+
   if (baseType === 'similar') {
     invariant(assertion.value, 'Similarity assertion must have a string value');
     invariant(
diff --git a/src/evaluator.ts b/src/evaluator.ts
index b53a4150d..4413cc819 100644
--- a/src/evaluator.ts
+++ b/src/evaluator.ts
@@ -3,6 +3,7 @@ import readline from 'readline';
 import async from 'async';
 import chalk from 'chalk';
 import nunjucks from 'nunjucks';
+import invariant from 'tiny-invariant';
 
 import logger from './logger';
 import telemetry from './telemetry';
@@ -110,18 +111,40 @@ class Evaluator {
       vars,
     };
 
+    let latencyMs = 0;
     try {
+      const startTime = Date.now();
       const response = await provider.callApi(renderedPrompt);
+      const endTime = Date.now();
+      latencyMs = endTime - startTime;
+
       const ret: EvaluateResult = {
         ...setup,
         response,
         success: false,
         score: 0,
+        latencyMs,
       };
       if (response.error) {
         ret.error = response.error;
       } else if (response.output) {
-        const checkResult = await runAssertions(test, response.output);
+        // Create a copy of response so we can potentially mutate it.
+        let processedResponse = { ...response };
+        if (test.options?.postprocess) {
+          const { postprocess } = test.options;
+          const postprocessFn = new Function(
+            'output',
+            'context',
+            postprocess.includes('\n') ? postprocess : `return ${postprocess}`,
+          );
+          processedResponse.output = postprocessFn(processedResponse.output);
+          if (processedResponse.output == null) {
+            throw new Error('Postprocess function did not return a value');
+          }
+        }
+
+        invariant(processedResponse.output != null, 'Response output should not be null');
+        const checkResult = await runAssertions(test, processedResponse.output);
         if (!checkResult.pass) {
           ret.error = checkResult.reason;
         }
@@ -132,6 +155,7 @@ class Evaluator {
           this.stats.tokenUsage.prompt += checkResult.tokensUsed.prompt;
           this.stats.tokenUsage.completion += checkResult.tokensUsed.completion;
         }
+        ret.response = processedResponse;
       } else {
         ret.success = false;
         ret.score = 0;
@@ -159,6 +183,7 @@ class Evaluator {
         error: String(err) + '\n\n' + (err as Error).stack,
         success: false,
         score: 0,
+        latencyMs,
       };
     }
   }
@@ -232,11 +257,13 @@ class Evaluator {
     // Aggregate all vars across test cases
 
     const tests = (
-      testSuite.tests || [
-        {
-          // Dummy test for cases when we're only comparing raw prompts.
-        },
-      ]
+      testSuite.tests && testSuite.tests.length > 0
+        ? testSuite.tests
+        : [
+            {
+              // Dummy test for cases when we're only comparing raw prompts.
+            },
+          ]
     ).map((test) => {
       const finalTestCase: TestCase = Object.assign({}, testSuite.defaultTest);
       return Object.assign(finalTestCase, test);
@@ -270,6 +297,8 @@ class Evaluator {
         testCase.options?.prefix || testSuite.defaultTest?.options?.prefix || '';
       const appendToPrompt =
         testCase.options?.suffix || testSuite.defaultTest?.options?.suffix || '';
+      testCase.options.postprocess =
+        testCase.options.postprocess || testSuite.defaultTest?.options?.postprocess;
 
       // Finalize test case eval
       const varCombinations = generateVarCombinations(testCase.vars || {});
@@ -404,6 +433,8 @@ class Evaluator {
           score: row.score,
           text: resultText,
           prompt: row.prompt.raw,
+          latencyMs: row.latencyMs,
+          tokenUsage: row.response?.tokenUsage,
         };
       },
     );
diff --git a/src/index.ts b/src/index.ts
index a9bb7e164..08156118a 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -3,7 +3,7 @@ import providers from './providers';
 import telemetry from './telemetry';
 import { evaluate as doEvaluate } from './evaluator';
 import { loadApiProviders } from './providers';
-import { readTests } from './util';
+import { readTests, writeOutput } from './util';
 import type { EvaluateOptions, TestSuite, TestSuiteConfig } from './types';
 
 export * from './types';
@@ -28,6 +28,11 @@ async function evaluate(testSuite: EvaluateTestSuite, options: EvaluateOptions =
   };
   telemetry.maybeShowNotice();
   const ret = await doEvaluate(constructedTestSuite, options);
+
+  if (testSuite.outputPath) {
+    writeOutput(testSuite.outputPath, ret, testSuite, null);
+  }
+
   await telemetry.send();
   return ret;
 }
diff --git a/src/main.ts b/src/main.ts
index 910b21ad3..6f63f3323 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -11,6 +11,7 @@ import logger, { getLogLevel, setLogLevel } from './logger';
 import { loadApiProvider, loadApiProviders } from './providers';
 import { evaluate } from './evaluator';
 import {
+  cleanupOldResults,
   maybeReadConfig,
   readConfig,
   readLatestResults,
@@ -181,6 +182,7 @@ async function main() {
     .action(async () => {
       telemetry.maybeShowNotice();
       await clearCache();
+      cleanupOldResults(0);
       telemetry.record('command_used', {
         name: 'cache_clear',
       });
@@ -321,6 +323,7 @@ async function main() {
           suffix: cmdObj.promptSuffix,
           provider: cmdObj.grader,
           // rubricPrompt:
+          // postprocess
         },
         ...config.defaultTest,
       };
diff --git a/src/providers/azureopenai.ts b/src/providers/azureopenai.ts
index aad04fbe6..6ad9e4051 100644
--- a/src/providers/azureopenai.ts
+++ b/src/providers/azureopenai.ts
@@ -6,6 +6,10 @@ import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '.
 
 interface AzureOpenAiCompletionOptions {
   temperature?: number;
+  top_p?: number;
+  frequency_penalty?: number;
+  presence_penalty?: number;
+  best_of?: number;
   functions?: {
     name: string;
     description?: string;
@@ -144,6 +148,17 @@ export class AzureOpenAiCompletionProvider extends AzureOpenAiGenericProvider {
         options?.temperature ??
         this.options.temperature ??
         parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
+      presence_penalty:
+        options?.presence_penalty ??
+        this.options.presence_penalty ??
+        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
+      frequency_penalty:
+        options?.frequency_penalty ??
+        this.options.frequency_penalty ??
+        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
+      best_of:
+        options?.best_of ?? this.options.best_of ?? parseInt(process.env.OPENAI_BEST_OF || '1'),
       stop,
     };
     logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
@@ -214,6 +229,15 @@ export class AzureOpenAiChatCompletionProvider extends AzureOpenAiGenericProvide
         options?.temperature ??
         this.options.temperature ??
         parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
+      presence_penalty:
+        options?.presence_penalty ??
+        this.options.presence_penalty ??
+        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
+      frequency_penalty:
+        options?.frequency_penalty ??
+        this.options.frequency_penalty ??
+        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
       functions: options?.functions || this.options.functions || undefined,
       function_call: options?.function_call || this.options.function_call || undefined,
     };
diff --git a/src/providers/openai.ts b/src/providers/openai.ts
index ed68b6242..9652c4011 100644
--- a/src/providers/openai.ts
+++ b/src/providers/openai.ts
@@ -1,4 +1,3 @@
-
 import logger from '../logger';
 import { fetchJsonWithCache } from '../cache';
 import { REQUEST_TIMEOUT_MS, parseChatPrompt } from './shared';
@@ -10,6 +9,10 @@ const DEFAULT_OPENAI_HOST = 'api.openai.com';
 interface OpenAiCompletionOptions {
   temperature?: number;
   max_tokens?: number;
+  top_p?: number;
+  frequency_penalty?: number;
+  presence_penalty?: number;
+  best_of?: number;
   functions?: {
     name: string;
     description?: string;
@@ -148,11 +151,25 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
     const body = {
       model: this.modelName,
       prompt,
-      max_tokens: options?.max_tokens ?? this.options.max_tokens ?? parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      max_tokens:
+        options?.max_tokens ??
+        this.options.max_tokens ??
+        parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
       temperature:
         options?.temperature ??
         this.options.temperature ??
         parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
+      presence_penalty:
+        options?.presence_penalty ??
+        this.options.presence_penalty ??
+        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
+      frequency_penalty:
+        options?.frequency_penalty ??
+        this.options.frequency_penalty ??
+        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
+      best_of:
+        options?.best_of ?? this.options.best_of ?? parseInt(process.env.OPENAI_BEST_OF || '1'),
       stop,
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
@@ -231,11 +248,23 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     const body = {
       model: this.modelName,
       messages: messages,
-      max_tokens: options?.max_tokens ?? this.options.max_tokens ?? parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      max_tokens:
+        options?.max_tokens ??
+        this.options.max_tokens ??
+        parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
       temperature:
         options?.temperature ??
         this.options.temperature ??
         parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
+      presence_penalty:
+        options?.presence_penalty ??
+        this.options.presence_penalty ??
+        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
+      frequency_penalty:
+        options?.frequency_penalty ??
+        this.options.frequency_penalty ??
+        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
       functions: options?.functions || this.options.functions || undefined,
       function_call: options?.function_call || this.options.function_call || undefined,
     };
diff --git a/src/providers/replicate.ts b/src/providers/replicate.ts
index 8a2fcc3c5..70d030c8a 100644
--- a/src/providers/replicate.ts
+++ b/src/providers/replicate.ts
@@ -65,9 +65,13 @@ export class ReplicateProvider implements ApiProvider {
       const data = {
         input: {
           prompt,
-          max_length: this.options.max_length || parseInt(process.env.REPLICATE_MAX_LENGTH || '2046', 10),
-          temperature: this.options.temperature || parseFloat(process.env.REPLICATE_TEMPERATURE || '0.01'),
-          repetition_penalty: this.options.repetition_penalty || parseFloat(process.env.REPLICATE_REPETITION_PENALTY || '1.0'),
+          max_length:
+            this.options.max_length || parseInt(process.env.REPLICATE_MAX_LENGTH || '2046', 10),
+          temperature:
+            this.options.temperature || parseFloat(process.env.REPLICATE_TEMPERATURE || '0.01'),
+          repetition_penalty:
+            this.options.repetition_penalty ||
+            parseFloat(process.env.REPLICATE_REPETITION_PENALTY || '1.0'),
         },
       };
       response = await replicate.run(this.modelName as any, data);
diff --git a/src/providers/shared.ts b/src/providers/shared.ts
index a17f89474..12f9906cc 100644
--- a/src/providers/shared.ts
+++ b/src/providers/shared.ts
@@ -4,7 +4,9 @@ export const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
   ? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
   : 300_000;
 
-export function parseChatPrompt(prompt: string): { role: string; content: string; name?: string }[] {
+export function parseChatPrompt(
+  prompt: string,
+): { role: string; content: string; name?: string }[] {
   const trimmedPrompt = prompt.trim();
   if (trimmedPrompt.startsWith('- role:')) {
     try {
diff --git a/src/types.ts b/src/types.ts
index 53431ffd0..3a3b2e5de 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -73,6 +73,10 @@ export interface PromptConfig {
   suffix?: string;
 }
 
+export interface OutputConfig {
+  postprocess?: string;
+}
+
 export interface EvaluateOptions {
   maxConcurrency?: number;
   showProgressBar?: boolean;
@@ -92,6 +96,7 @@ export interface EvaluateResult {
   error?: string;
   success: boolean;
   score: number;
+  latencyMs: number;
 }
 
 export interface EvaluateTableOutput {
@@ -99,6 +104,8 @@ export interface EvaluateTableOutput {
   score: number;
   text: string;
   prompt: string;
+  latencyMs: number;
+  tokenUsage?: Partial<TokenUsage>;
 }
 
 export interface EvaluateTable {
@@ -144,6 +151,7 @@ type BaseAssertionTypes =
   | 'is-json'
   | 'contains-json'
   | 'javascript'
+  | 'python'
   | 'similar'
   | 'llm-rubric'
   | 'webhook'
@@ -181,11 +189,14 @@ export interface TestCase {
   // Key-value pairs to substitute in the prompt
   vars?: Record<string, string | string[] | object>;
 
+  // Optional filepath or glob pattern to load vars from
+  loadVars?: string | string[];
+
   // Optional list of automatic checks to run on the LLM output
   assert?: Assertion[];
 
   // Additional configuration settings for the prompt
-  options?: PromptConfig & GradingConfig;
+  options?: PromptConfig & OutputConfig & GradingConfig;
 }
 
 // Same as a TestCase, except the `vars` object has been flattened into its final form.
@@ -231,7 +242,7 @@ export interface TestSuiteConfig {
   prompts: string | string[];
 
   // Path to a test file, OR list of LLM prompt variations (aka "test case")
-  tests: string | TestCase[];
+  tests: string | string[] | TestCase[];
 
   // Sets the default properties for each test case. Useful for setting an assertion, on all test cases, for example.
   defaultTest?: Omit<TestCase, 'description'>;
diff --git a/src/util.ts b/src/util.ts
index 703fe799b..bc5b369f3 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -29,7 +29,10 @@ import type {
   TestSuite,
 } from './types';
 
-export function readProviderPromptMap(config: Partial<UnifiedConfig>, parsedPrompts: Prompt[]): TestSuite["providerPromptMap"] {
+export function readProviderPromptMap(
+  config: Partial<UnifiedConfig>,
+  parsedPrompts: Prompt[],
+): TestSuite['providerPromptMap'] {
   const ret: Record<string, string[]> = {};
 
   if (!config.providers) {
@@ -221,7 +224,31 @@ export async function fetchCsvFromGoogleSheet(url: string): Promise<string> {
   return csvData;
 }
 
-export async function readVars(varsPath: string, basePath: string = ''): Promise<CsvRow[]> {
+export async function readVarsFiles(
+  pathOrGlobs: string | string[],
+  basePath: string = '',
+): Promise<Record<string, string | string[] | object>> {
+  if (typeof pathOrGlobs === 'string') {
+    pathOrGlobs = [pathOrGlobs];
+  }
+
+  const ret: Record<string, string | string[] | object> = {};
+  for (const pathOrGlob of pathOrGlobs) {
+    const resolvedPath = path.resolve(basePath, pathOrGlob);
+    const paths = globSync(resolvedPath);
+
+    for (const p of paths) {
+      const yamlData = yaml.load(fs.readFileSync(p, 'utf-8'));
+      Object.assign(ret, yamlData);
+    }
+  }
+
+  return ret;
+}
+
+export async function readTestsFile(varsPath: string, basePath: string = ''): Promise<CsvRow[]> {
+  // This function is confusingly named - it reads a CSV, JSON, or YAML file of
+  // TESTS or test equivalents.
   const resolvedVarsPath = path.resolve(basePath, varsPath);
   const fileExtension = parsePath(varsPath).ext.slice(1);
   let rows: CsvRow[] = [];
@@ -243,25 +270,53 @@ export async function readVars(varsPath: string, basePath: string = ''): Promise
 }
 
 export async function readTests(
-  tests: string | TestCase[] | undefined,
+  tests: string | string[] | TestCase[] | undefined,
   basePath: string = '',
 ): Promise<TestCase[]> {
-  if (!tests) {
-    return [];
-  }
+  const ret: TestCase[] = [];
+
+  const loadTestsFromGlob = async (loadTestsGlob: string) => {
+    const resolvedPath = path.resolve(basePath, loadTestsGlob);
+    const testFiles = globSync(resolvedPath);
+    for (const testFile of testFiles) {
+      const testFileContent = yaml.load(fs.readFileSync(testFile, 'utf-8')) as TestCase[];
+      for (const testCase of testFileContent) {
+        if (typeof testCase.vars === 'string' || Array.isArray(testCase.vars)) {
+          const testcaseBasePath = path.dirname(testFile);
+          testCase.vars = await readVarsFiles(testCase.vars, testcaseBasePath);
+        }
+      }
+      ret.push(...testFileContent);
+    }
+  };
 
   if (typeof tests === 'string') {
-    // It's a filepath, load from CSV
-    const vars = await readVars(tests, basePath);
-    return vars.map((row, idx) => {
-      const test = testCaseFromCsvRow(row);
-      test.description = `Row #${idx + 1}`;
-      return test;
-    });
+    if (tests.endsWith('yaml') || tests.endsWith('yml')) {
+      // Load testcase config from yaml
+      await loadTestsFromGlob(tests);
+    } else {
+      // Legacy load CSV
+      const vars = await readTestsFile(tests, basePath);
+      return vars.map((row, idx) => {
+        const test = testCaseFromCsvRow(row);
+        test.description = `Row #${idx + 1}`;
+        return test;
+      });
+    }
+  } else if (Array.isArray(tests)) {
+    for (const maybeTestsGlob of tests) {
+      if (typeof maybeTestsGlob === 'string') {
+        // Assume it's a filepath
+        await loadTestsFromGlob(maybeTestsGlob);
+      } else {
+        // Assume it's a full test case
+        ret.push(maybeTestsGlob);
+      }
+    }
   }
 
   // Some validation of the shape of tests
-  for (const test of tests) {
+  for (const test of ret) {
     if (!test.assert && !test.vars) {
       throw new Error(
         `Test case must have either "assert" or "vars" property. Instead got ${JSON.stringify(
@@ -273,7 +328,7 @@ export async function readTests(
     }
   }
 
-  return tests;
+  return ret;
 }
 
 export function writeOutput(
@@ -313,28 +368,31 @@ export function writeOutput(
   }
 }
 
-export async function fetchWithTimeout(
+export function fetchWithTimeout(
   url: RequestInfo,
   options: RequestInit = {},
   timeout: number,
 ): Promise<Response> {
-  const controller = new AbortController();
-  const { signal } = controller;
-  options.signal = signal;
-
-  const timeoutId = setTimeout(() => {
-    controller.abort();
-    throw new Error(`Request timed out after ${timeout} ms`);
-  }, timeout);
-
-  try {
-    const response = await fetch(url, options);
-    clearTimeout(timeoutId);
-    return response;
-  } catch (error) {
-    clearTimeout(timeoutId);
-    throw error;
-  }
+  return new Promise((resolve, reject) => {
+    const controller = new AbortController();
+    const { signal } = controller;
+    options.signal = signal;
+
+    const timeoutId = setTimeout(() => {
+      controller.abort();
+      reject(new Error(`Request timed out after ${timeout} ms`));
+    }, timeout);
+
+    fetch(url, options)
+      .then((response) => {
+        clearTimeout(timeoutId);
+        resolve(response);
+      })
+      .catch((error) => {
+        clearTimeout(timeoutId);
+        reject(error);
+      });
+  });
 }
 
 export async function fetchWithRetries(
@@ -356,6 +414,8 @@ export async function fetchWithRetries(
   throw new Error(`Request failed after ${retries} retries: ${(lastError as Error).message}`);
 }
 
+const RESULT_HISTORY_LENGTH = 50;
+
 export function getConfigDirectoryPath(): string {
   return path.join(os.homedir(), '.promptfoo');
 }
@@ -365,11 +425,17 @@ export function getLatestResultsPath(): string {
 }
 
 export function writeLatestResults(results: EvaluateSummary, config: Partial<UnifiedConfig>) {
+  const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
+
+  // Replace hyphens with colons (Windows compatibility).
+  const timestamp = new Date().toISOString().replace(/:/g, '-');
+
+  const newResultsPath = path.join(resultsDirectory, `eval-${timestamp}.json`);
   const latestResultsPath = getLatestResultsPath();
   try {
-    fs.mkdirSync(path.dirname(latestResultsPath), { recursive: true });
+    fs.mkdirSync(resultsDirectory, { recursive: true });
     fs.writeFileSync(
-      latestResultsPath,
+      newResultsPath,
       JSON.stringify(
         {
           version: 1,
@@ -380,8 +446,45 @@ export function writeLatestResults(results: EvaluateSummary, config: Partial<Uni
         2,
       ),
     );
+    if (fs.existsSync(latestResultsPath)) {
+      fs.unlinkSync(latestResultsPath);
+    }
+    fs.symlinkSync(newResultsPath, latestResultsPath);
+    cleanupOldResults();
   } catch (err) {
-    logger.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
+    logger.error(`Failed to write latest results to ${newResultsPath}:\n${err}`);
+  }
+}
+
+export function listPreviousResults(): string[] {
+  const directory = path.join(getConfigDirectoryPath(), 'output');
+  const files = fs.readdirSync(directory);
+  const resultsFiles = files.filter((file) => file.startsWith('eval-') && file.endsWith('.json'));
+  const sortedFiles = resultsFiles.sort((a, b) => {
+    const statA = fs.statSync(path.join(directory, a));
+    const statB = fs.statSync(path.join(directory, b));
+    return statB.birthtime.getTime() - statA.birthtime.getTime(); // sort in descending order
+  });
+  return sortedFiles;
+}
+
+export function cleanupOldResults(remaining = RESULT_HISTORY_LENGTH) {
+  const sortedFiles = listPreviousResults();
+  for (let i = 0; i < sortedFiles.length - remaining; i++) {
+    fs.unlinkSync(path.join(getConfigDirectoryPath(), 'output', sortedFiles[i]));
+  }
+}
+
+export function readResult(
+  name: string,
+): { results: EvaluateSummary; config: Partial<UnifiedConfig> } | undefined {
+  const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
+  const resultsPath = path.join(resultsDirectory, name);
+  try {
+    const results = JSON.parse(fs.readFileSync(fs.realpathSync(resultsPath), 'utf-8'));
+    return results;
+  } catch (err) {
+    logger.error(`Failed to read results from ${resultsPath}:\n${err}`);
   }
 }
 
@@ -389,12 +492,7 @@ export function readLatestResults():
   | { results: EvaluateSummary; config: Partial<UnifiedConfig> }
   | undefined {
   const latestResultsPath = getLatestResultsPath();
-  try {
-    const latestResults = JSON.parse(fs.readFileSync(latestResultsPath, 'utf-8'));
-    return latestResults;
-  } catch (err) {
-    logger.error(`Failed to read latest results from ${latestResultsPath}:\n${err}`);
-  }
+  return readResult(latestResultsPath);
 }
 
 export function cosineSimilarity(vecA: number[], vecB: number[]) {
diff --git a/src/web/client/src/App.tsx b/src/web/client/src/App.tsx
index 35dedd331..1e2cf11b5 100644
--- a/src/web/client/src/App.tsx
+++ b/src/web/client/src/App.tsx
@@ -14,6 +14,7 @@ function App() {
   const { table, setTable, setConfig } = useStore();
   const [loaded, setLoaded] = React.useState<boolean>(false);
   const loadedFromApi = React.useRef(false);
+  const [recentFiles, setRecentFiles] = React.useState<string[]>([]);
 
   const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
   const [darkMode, setDarkMode] = React.useState(prefersDarkMode);
@@ -43,6 +44,22 @@ function App() {
     }
   }, [prefersDarkMode]);
 
+  const fetchRecentFiles = async () => {
+    if (!window.location.href.includes('localhost')) {
+      return;
+    }
+    const resp = await fetch(`http://localhost:15500/results`);
+    const body = await resp.json();
+    setRecentFiles(body.data);
+  };
+
+  const handleRecentFileSelection = async (file: string) => {
+    const resp = await fetch(`http://localhost:15500/results/${file}`);
+    const body = await resp.json();
+    setTable(body.data.results.table);
+    setConfig(body.data.config);
+  };
+
   React.useEffect(() => {
     const fetchEvalData = async (id: string) => {
       if (loadedFromApi.current) {
@@ -72,12 +89,14 @@ function App() {
         setLoaded(true);
         setTable(data.results.table);
         setConfig(data.config);
+        fetchRecentFiles();
       });
 
       socket.on('update', (data) => {
         console.log('Received data update', data);
         setTable(data.results.table);
         setConfig(data.config);
+        fetchRecentFiles();
       });
     }
 
@@ -89,7 +108,11 @@ function App() {
   return (
     <ThemeProvider theme={theme}>
       <NavBar darkMode={darkMode} onToggleDarkMode={toggleDarkMode} />
-      {loaded && table ? <ResultsView /> : <div>Loading...</div>}
+      {loaded && table ? (
+        <ResultsView recentFiles={recentFiles} onRecentFileSelected={handleRecentFileSelection} />
+      ) : (
+        <div>Loading...</div>
+      )}
     </ThemeProvider>
   );
 }
diff --git a/src/web/client/src/ResultsTable.css b/src/web/client/src/ResultsTable.css
index c86c96906..bbd9821a5 100644
--- a/src/web/client/src/ResultsTable.css
+++ b/src/web/client/src/ResultsTable.css
@@ -83,7 +83,17 @@ tr .cell-actions {
   font-size: 1.75rem;
 }
 
-tr:hover .cell-actions {
+tr .cell-detail {
+  visibility: hidden;
+  position: absolute;
+  bottom: 0.25rem;
+  margin-top: 1rem;
+  font-size: 0.75rem;
+  color: #888;
+}
+
+tr:hover .cell-actions,
+tr:hover .cell-detail {
   visibility: visible;
 }
 
diff --git a/src/web/client/src/ResultsTable.tsx b/src/web/client/src/ResultsTable.tsx
index e737820de..8b5053619 100644
--- a/src/web/client/src/ResultsTable.tsx
+++ b/src/web/client/src/ResultsTable.tsx
@@ -135,6 +135,16 @@ function EvalOutputCell({
         )}{' '}
         <TruncatedText text={text} maxLength={maxTextLength} />
       </div>
+      <div className="cell-detail">
+        {output.tokenUsage?.cached ? (
+          <span>{output.tokenUsage.cached} tokens (cached)</span>
+        ) : (
+          <>
+            {output.tokenUsage?.total && <span>{output.tokenUsage.total} tokens</span>} |{' '}
+            <span>{output.latencyMs} ms</span>
+          </>
+        )}
+      </div>
       <div className="cell-actions">
         {output.prompt && (
           <>
diff --git a/src/web/client/src/ResultsView.tsx b/src/web/client/src/ResultsView.tsx
index 7566bf012..afafddfe3 100644
--- a/src/web/client/src/ResultsView.tsx
+++ b/src/web/client/src/ResultsView.tsx
@@ -37,7 +37,32 @@ const ResponsiveStack = styled(Stack)(({ theme }) => ({
   },
 }));
 
-export default function ResultsView() {
+function filenameToDate(filename: string) {
+  const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
+
+  // Replace hyphens with colons where necessary (Windows compatibility).
+  const dateParts = dateString.split('T');
+  const timePart = dateParts[1].replace(/-/g, ':');
+  const formattedDateString = `${dateParts[0]}T${timePart}`;
+
+  const date = new Date(formattedDateString);
+  return date.toLocaleDateString('en-US', {
+    year: 'numeric',
+    month: 'long',
+    day: 'numeric',
+    hour: '2-digit',
+    minute: '2-digit',
+    second: '2-digit',
+    timeZoneName: 'short',
+  });
+}
+
+interface ResultsViewProps {
+  recentFiles: string[];
+  onRecentFileSelected: (file: string) => void;
+}
+
+export default function ResultsView({ recentFiles, onRecentFileSelected }: ResultsViewProps) {
   const { table, config } = useStore();
   const [maxTextLength, setMaxTextLength] = React.useState(250);
   const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({});
@@ -148,10 +173,30 @@ export default function ResultsView() {
   return (
     <div>
       <Paper py="md">
-        <ResponsiveStack direction="row" spacing={8} alignItems="center">
+        <ResponsiveStack direction="row" spacing={4} alignItems="center">
+          <Box>
+            {recentFiles && recentFiles.length > 0 && (
+              <FormControl sx={{ m: 1, minWidth: 200 }} size="small">
+                <InputLabel>View run</InputLabel>
+                <Select
+                  key={recentFiles.join(',')}
+                  className="recent-files"
+                  label="Previous runs"
+                  defaultValue={recentFiles[0]}
+                  onChange={(e: SelectChangeEvent) => onRecentFileSelected(e.target.value)}
+                >
+                  {recentFiles.map((file) => (
+                    <MenuItem key={file} value={file}>
+                      {filenameToDate(file)}
+                    </MenuItem>
+                  ))}
+                </Select>
+              </FormControl>
+            )}
+          </Box>
           <Box>
             <FormControl sx={{ m: 1, minWidth: 200 }} size="small">
-              <InputLabel id="visible-columns-label">Visible columns</InputLabel>
+              <InputLabel id="visible-columns-label">Show columns</InputLabel>
               <Select
                 labelId="visible-columns-label"
                 id="visible-columns"
diff --git a/src/web/client/src/types.ts b/src/web/client/src/types.ts
index 64b0ef09e..780f43694 100644
--- a/src/web/client/src/types.ts
+++ b/src/web/client/src/types.ts
@@ -1,3 +1,5 @@
+import { TokenUsage } from '../../../types';
+
 type Prompt = {
   display: string;
   raw: string;
@@ -13,6 +15,8 @@ export type EvalRowOutput = {
   score: number;
   text: string | object;
   prompt: string;
+  latencyMs: number;
+  tokenUsage?: Partial<TokenUsage>;
 };
 
 export type EvalRow = {
diff --git a/src/web/server.ts b/src/web/server.ts
index 7a2cfa7c0..d3623d434 100644
--- a/src/web/server.ts
+++ b/src/web/server.ts
@@ -1,4 +1,4 @@
-import fs from 'fs';
+import fs, { Stats } from 'fs';
 import path from 'node:path';
 import readline from 'node:readline';
 import http from 'node:http';
@@ -11,7 +11,7 @@ import { Server as SocketIOServer } from 'socket.io';
 
 import logger from '../logger';
 import { getDirectory } from '../esm';
-import { getLatestResultsPath } from '../util';
+import { getLatestResultsPath, listPreviousResults, readResult } from '../util';
 
 export function init(port = 15500) {
   const app = express();
@@ -40,14 +40,37 @@ export function init(port = 15500) {
     socket.emit('init', readLatestJson());
 
     // Watch for changes to latest.json and emit the update event
-    fs.watch(
-      latestJsonPath,
-      debounce((event: string) => {
-        if (event === 'change') {
-          socket.emit('update', readLatestJson());
-        }
-      }, 250),
-    );
+    const watcher = debounce((curr: Stats, prev: Stats) => {
+      if (curr.mtime !== prev.mtime) {
+        socket.emit('update', readLatestJson());
+      }
+    }, 250);
+    fs.watchFile(latestJsonPath, watcher);
+
+    // Stop watching the file when the socket connection is closed
+    socket.on('disconnect', () => {
+      fs.unwatchFile(latestJsonPath, watcher);
+    });
+  });
+
+  app.get('/results', (req, res) => {
+    const previousResults = listPreviousResults();
+    res.json({ data: previousResults });
+  });
+
+  app.get('/results/:filename', (req, res) => {
+    const filename = req.params.filename;
+    const safeFilename = path.basename(filename);
+    if (safeFilename !== filename || !listPreviousResults().includes(safeFilename)) {
+      res.status(400).send('Invalid filename');
+      return;
+    }
+    const result = readResult(safeFilename);
+    if (!result) {
+      res.status(404).send('Result not found');
+      return;
+    }
+    res.json({ data: result });
   });
 
   httpServer.listen(port, () => {


Test patch:
diff --git a/examples/separate-test-configs/README.md b/examples/separate-test-configs/README.md
new file mode 100644
index 000000000..4fdf54e91
--- /dev/null
+++ b/examples/separate-test-configs/README.md
@@ -0,0 +1,7 @@
+This example shows how you can use paths and wildcards to simplify for `promptfooconfig.yaml`.
+
+Run:
+
+```
+promptfoo eval
+```
diff --git a/examples/separate-test-configs/promptfooconfig.yaml b/examples/separate-test-configs/promptfooconfig.yaml
new file mode 100644
index 000000000..0b5dfd3cc
--- /dev/null
+++ b/examples/separate-test-configs/promptfooconfig.yaml
@@ -0,0 +1,20 @@
+prompts: prompts.txt
+providers: openai:gpt-3.5-turbo
+
+# The defaultTest configuration is applied to every test case in this config.
+defaultTest:
+  options:
+    suffix: Be extremely concise
+
+# Loads & runs all test cases matching these filepaths
+tests:
+  # You can supply an exact filepath
+  - tests/tests2.yaml
+
+  # Or a glob (wildcard)
+  - tests/*
+
+  # Mix and match with actual test cases
+  - vars:
+      topic: the internet
+      content_type: witty tweets
diff --git a/examples/separate-test-configs/prompts.txt b/examples/separate-test-configs/prompts.txt
new file mode 100644
index 000000000..f8fcb1683
--- /dev/null
+++ b/examples/separate-test-configs/prompts.txt
@@ -0,0 +1,3 @@
+Generate a list of creative {{content_type}} about {{topic}}.
+---
+Imagine you are an expert in {{topic}}. What {{content_type}} would you recommend writing about?
diff --git a/examples/separate-test-configs/tests/tests1.yaml b/examples/separate-test-configs/tests/tests1.yaml
new file mode 100644
index 000000000..07aed53d1
--- /dev/null
+++ b/examples/separate-test-configs/tests/tests1.yaml
@@ -0,0 +1,24 @@
+# This file contains a list of test cases. They're added alongside test cases
+# from tests2.yaml in the example promptfooconfig.yaml.
+
+- vars:
+    topic: artificial intelligence
+    content_type: blog post ideas
+  assert:
+    - type: javascript
+      value: output.length > 100
+- vars:
+    topic: climate change
+    content_type: policy proposals
+- vars:
+    topic: vegetarian cooking
+    content_type: recipe ideas
+- vars:
+    topic: cybersecurity
+    content_type: educational topics
+- vars:
+    topic: remote work
+    content_type: challenges
+- vars:
+    topic: quantum computing
+    content_type: use cases
diff --git a/examples/separate-test-configs/tests/tests2.yaml b/examples/separate-test-configs/tests/tests2.yaml
new file mode 100644
index 000000000..364cb436f
--- /dev/null
+++ b/examples/separate-test-configs/tests/tests2.yaml
@@ -0,0 +1,25 @@
+# This file contains a list of test cases. They're added alongside test cases
+# from tests1.yaml in the example promptfooconfig.yaml.
+
+- vars:
+    topic: alien invasions
+    content_type: survival tips
+  assert:
+    - type: javascript
+      value: output.length > 100
+- vars:
+    topic: time travel
+    content_type: tourist attractions
+- vars:
+    topic: zombie apocalypse
+    content_type: home renovation ideas
+- vars:
+    topic: unicorn breeding
+    content_type: best practices
+- vars:
+    topic: telepathy for beginners
+    content_type: etiquette rules
+- vars:
+    topic: fairy dust
+    content_type: cleaning hacks
+- vars: ../vars/vars_*.yaml
diff --git a/examples/separate-test-configs/vars/vars_extra.yaml b/examples/separate-test-configs/vars/vars_extra.yaml
new file mode 100644
index 000000000..b73272351
--- /dev/null
+++ b/examples/separate-test-configs/vars/vars_extra.yaml
@@ -0,0 +1,2 @@
+topic: bananas
+content_type: musings of an evil genius
diff --git a/test/evaluator.test.ts b/test/evaluator.test.ts
index 91de001f2..945b38f4b 100644
--- a/test/evaluator.test.ts
+++ b/test/evaluator.test.ts
@@ -345,12 +345,58 @@ describe('evaluator', () => {
     expect(summary.results[0].response?.output).toBe('Test output');
   });
 
+  test('evaluate with postprocess option - default test', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      defaultTest: {
+        options: {
+          postprocess: 'output + " postprocessed"',
+        },
+      },
+    };
+
+    const summary = await evaluate(testSuite, {});
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Test output postprocessed');
+  });
+
+  test('evaluate with postprocess option - single test', async () => {
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt')],
+      tests: [
+        {
+          assert: [
+            {
+              type: 'equals',
+              value: 'Test output postprocessed',
+            },
+          ],
+          options: {
+            postprocess: 'output + " postprocessed"',
+          },
+        },
+      ],
+    };
+
+    const summary = await evaluate(testSuite, {});
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    expect(summary.stats.successes).toBe(1);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Test output postprocessed');
+  });
+
   test('evaluate with providerPromptMap', async () => {
     const testSuite: TestSuite = {
       providers: [mockApiProvider],
       prompts: [toPrompt('Test prompt 1'), toPrompt('Test prompt 2')],
       providerPromptMap: {
-        'test-provider': ['Test prompt 1']
+        'test-provider': ['Test prompt 1'],
       },
       tests: [
         {
diff --git a/test/util.test.ts b/test/util.test.ts
index 2b1fc6783..77b074043 100644
--- a/test/util.test.ts
+++ b/test/util.test.ts
@@ -5,7 +5,7 @@ import yaml from 'js-yaml';
 import { globSync } from 'glob';
 
 import {
-  readVars,
+  readTestsFile,
   readPrompts,
   writeOutput,
   readTests,
@@ -137,7 +137,7 @@ describe('util', () => {
     (fs.readFileSync as jest.Mock).mockReturnValue('var1,var2\nvalue1,value2');
     const varsPath = 'vars.csv';
 
-    const result = await readVars(varsPath);
+    const result = await readTestsFile(varsPath);
 
     expect(fs.readFileSync).toHaveBeenCalledTimes(1);
     expect(result).toEqual([{ var1: 'value1', var2: 'value2' }]);
@@ -147,7 +147,7 @@ describe('util', () => {
     (fs.readFileSync as jest.Mock).mockReturnValue('[{"var1": "value1", "var2": "value2"}]');
     const varsPath = 'vars.json';
 
-    const result = await readVars(varsPath);
+    const result = await readTestsFile(varsPath);
 
     expect(fs.readFileSync).toHaveBeenCalledTimes(1);
     expect(result).toEqual([{ var1: 'value1', var2: 'value2' }]);
@@ -157,7 +157,7 @@ describe('util', () => {
     (fs.readFileSync as jest.Mock).mockReturnValue('- var1: value1\n  var2: value2');
     const varsPath = 'vars.yaml';
 
-    const result = await readVars(varsPath);
+    const result = await readTestsFile(varsPath);
 
     expect(fs.readFileSync).toHaveBeenCalledTimes(1);
     expect(result).toEqual([{ var1: 'value1', var2: 'value2' }]);
@@ -169,6 +169,7 @@ describe('util', () => {
       {
         success: true,
         score: 1.0,
+        latencyMs: 1000,
         prompt: {
           raw: 'Test prompt',
           display: '[display] Test prompt',
@@ -189,7 +190,9 @@ describe('util', () => {
       },
       body: [
         {
-          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          outputs: [
+            { pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt', latencyMs: 1000 },
+          ],
           vars: ['value1', 'value2'],
         },
       ],
@@ -224,6 +227,7 @@ describe('util', () => {
       {
         success: true,
         score: 1.0,
+        latencyMs: 1000,
         prompt: {
           raw: 'Test prompt',
           display: '[display] Test prompt',
@@ -244,7 +248,9 @@ describe('util', () => {
       },
       body: [
         {
-          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          outputs: [
+            { pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt', latencyMs: 1000 },
+          ],
           vars: ['value1', 'value2'],
         },
       ],
@@ -279,6 +285,7 @@ describe('util', () => {
       {
         success: true,
         score: 1.0,
+        latencyMs: 1000,
         prompt: {
           raw: 'Test prompt',
           display: '[display] Test prompt',
@@ -299,7 +306,9 @@ describe('util', () => {
       },
       body: [
         {
-          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          outputs: [
+            { pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt', latencyMs: 1000 },
+          ],
           vars: ['value1', 'value2'],
         },
       ],
@@ -438,4 +447,55 @@ describe('readTests', () => {
 
     expect(result).toEqual(input);
   });
+
+  test('readTests with string array input (paths to test configs)', async () => {
+    const testsPaths = ['test1.yaml', 'test2.yaml'];
+    const test1Content = [
+      {
+        description: 'Test 1',
+        vars: { var1: 'value1', var2: 'value2' },
+        assert: [{ type: 'equals', value: 'value1' }],
+      },
+    ];
+    const test2Content = [
+      {
+        description: 'Test 2',
+        vars: { var1: 'value3', var2: 'value4' },
+        assert: [{ type: 'contains-json', value: 'value3' }],
+      },
+    ];
+    (fs.readFileSync as jest.Mock)
+      .mockReturnValueOnce(yaml.dump(test1Content))
+      .mockReturnValueOnce(yaml.dump(test2Content));
+    (globSync as jest.Mock).mockImplementation((pathOrGlob) => [pathOrGlob]);
+
+    const result = await readTests(testsPaths);
+
+    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
+    expect(result).toEqual([...test1Content, ...test2Content]);
+  });
+
+  test('readTests with vars glob input (paths to vars configs)', async () => {
+    const testsPaths = ['test1.yaml'];
+    const test1Content = [
+      {
+        description: 'Test 1',
+        vars: 'vars1.yaml',
+        assert: [{ type: 'equals', value: 'value1' }],
+      },
+    ];
+    const vars1Content = {
+      var1: 'value1',
+      var2: 'value2',
+    };
+    (fs.readFileSync as jest.Mock)
+      .mockReturnValueOnce(yaml.dump(test1Content))
+      .mockReturnValueOnce(yaml.dump(vars1Content));
+    (globSync as jest.Mock).mockImplementation((pathOrGlob) => [pathOrGlob]);
+
+    const result = await readTests(testsPaths);
+
+    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
+    expect(result).toEqual([Object.assign({}, test1Content[0], {vars: vars1Content})]);
+  });
 });