Repository: promptfoo/promptfoo (ts)

Issue:
Support for loading test cases from directory
## Description

To facilitate easier management, discussion, and understanding of individual test cases, it would be beneficial to allow defining and loading tests (consisting of variables and assertions pairs) from a specified directory.

## Proposed Implementation

Consider a scenario where the test definitions could be mentioned in a configuration file in the following way:

promptfooconfig.yaml
```yaml
prompts: [prompts/prompt1.yaml]
providers: [openai:gpt-3.5-turbo]
tests: './tests/
```

Where the test directory would look like this:

```
tests
├── 1.yaml
├── 2.yaml
└── 3.yaml
```

Each yaml will then hold the defintion of the input vars and the assertions for one specific test case:

```yaml
vars:
  text: Some input text for the test
assert:
  - type: not-contains-any
    value:
      - "diagnose"
  - type: contains-all
    value:
      - "enigmatic creatures"
  - type: similar
    value: Some input text for the testing
    threshold: 0.95
```

## Potential Benefits:

**1. Granular Test Case Management:** This structure enables users to work with each test case as a separate unit, alleviating the necessity of maintaining all test cases within a single file.

**2. Facilitates Discussion:** A numerical test case system allows teams to easily refer and discuss specific test cases.

**3. Grouping Tests:** This structure would enable users to categorize their tests based on directory names (for example, tests/accuracy/ and tests/creativity/), enabling easier comparison of prompt performance across various performance categories, rather than on an aggregate measure across all tests.

Currently, while tests.csv is used, it does not allow defining multiple assertions for the same variable. This proposed change will be especially useful when the number of assertions for each test case is significantly large, making their management in separate files more efficient.

I'm open to contributing towards the implementation of this feature. Please let me know if any additional information is required.

Pull request description:
Add support for loading test cases from file/directory path
Lets you do things like this:

```yaml
prompts: prompts.txt
providers: openai:gpt-3.5-turbo

# Loads & runs all test cases matching these filepaths
tests:
  # You can supply an exact filepath
  - tests/tests2.yaml

  # Or a glob (wildcard)
  - tests/*

  # Mix and match with actual test cases
  - vars:
      var1: foo
      var2: bar
```

This is also valid:
```yaml
tests: tests/*
```

Or this:
```yaml
tests: ['tests/accuracy', 'tests/creativity']
```

----------------

Also adds support for importing vars, like this:

```yaml
tests:
  - vars: path/to/vars*.yaml
``` 

----------------

Closes #86 
Relevant to #73 

Patch:
diff --git a/README.md b/README.md
index 66b9fc092..9c280c00c 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
 
 [![npm](https://img.shields.io/npm/v/promptfoo)](https://npmjs.com/package/promptfoo)
 [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/typpo/promptfoo/main.yml)](https://github.com/typpo/promptfoo/actions/workflows/main.yml)
+![MIT license](https://img.shields.io/github/license/typpo/promptfoo)
 
 `promptfoo` is a tool for testing and evaluating LLM prompt quality.
 
@@ -102,6 +103,7 @@ See [Test assertions](https://promptfoo.dev/docs/configuration/expected-outputs)
 | `is-json`       | output is valid json                                                      |
 | `contains-json` | output contains valid json                                                |
 | `javascript`    | provided Javascript function validates the output                         |
+| `python`        | provided Python function validates the output                             |
 | `webhook`       | provided webhook returns `{pass: true}`                                   |
 | `similar`       | embeddings and cosine similarity are above a threshold                    |
 | `llm-rubric`    | LLM output matches a given rubric, using a Language Model to grade output |
diff --git a/src/types.ts b/src/types.ts
index d9034e742..23caa68d8 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -188,6 +188,9 @@ export interface TestCase {
   // Key-value pairs to substitute in the prompt
   vars?: Record<string, string | string[] | object>;
 
+  // Optional filepath or glob pattern to load vars from
+  loadVars?: string | string[];
+
   // Optional list of automatic checks to run on the LLM output
   assert?: Assertion[];
 
@@ -238,7 +241,7 @@ export interface TestSuiteConfig {
   prompts: string | string[];
 
   // Path to a test file, OR list of LLM prompt variations (aka "test case")
-  tests: string | TestCase[];
+  tests: string | string[] | TestCase[];
 
   // Sets the default properties for each test case. Useful for setting an assertion, on all test cases, for example.
   defaultTest?: Omit<TestCase, 'description'>;
diff --git a/src/util.ts b/src/util.ts
index f1753d045..bc5b369f3 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -224,7 +224,31 @@ export async function fetchCsvFromGoogleSheet(url: string): Promise<string> {
   return csvData;
 }
 
-export async function readVars(varsPath: string, basePath: string = ''): Promise<CsvRow[]> {
+export async function readVarsFiles(
+  pathOrGlobs: string | string[],
+  basePath: string = '',
+): Promise<Record<string, string | string[] | object>> {
+  if (typeof pathOrGlobs === 'string') {
+    pathOrGlobs = [pathOrGlobs];
+  }
+
+  const ret: Record<string, string | string[] | object> = {};
+  for (const pathOrGlob of pathOrGlobs) {
+    const resolvedPath = path.resolve(basePath, pathOrGlob);
+    const paths = globSync(resolvedPath);
+
+    for (const p of paths) {
+      const yamlData = yaml.load(fs.readFileSync(p, 'utf-8'));
+      Object.assign(ret, yamlData);
+    }
+  }
+
+  return ret;
+}
+
+export async function readTestsFile(varsPath: string, basePath: string = ''): Promise<CsvRow[]> {
+  // This function is confusingly named - it reads a CSV, JSON, or YAML file of
+  // TESTS or test equivalents.
   const resolvedVarsPath = path.resolve(basePath, varsPath);
   const fileExtension = parsePath(varsPath).ext.slice(1);
   let rows: CsvRow[] = [];
@@ -246,25 +270,53 @@ export async function readVars(varsPath: string, basePath: string = ''): Promise
 }
 
 export async function readTests(
-  tests: string | TestCase[] | undefined,
+  tests: string | string[] | TestCase[] | undefined,
   basePath: string = '',
 ): Promise<TestCase[]> {
-  if (!tests) {
-    return [];
-  }
+  const ret: TestCase[] = [];
+
+  const loadTestsFromGlob = async (loadTestsGlob: string) => {
+    const resolvedPath = path.resolve(basePath, loadTestsGlob);
+    const testFiles = globSync(resolvedPath);
+    for (const testFile of testFiles) {
+      const testFileContent = yaml.load(fs.readFileSync(testFile, 'utf-8')) as TestCase[];
+      for (const testCase of testFileContent) {
+        if (typeof testCase.vars === 'string' || Array.isArray(testCase.vars)) {
+          const testcaseBasePath = path.dirname(testFile);
+          testCase.vars = await readVarsFiles(testCase.vars, testcaseBasePath);
+        }
+      }
+      ret.push(...testFileContent);
+    }
+  };
 
   if (typeof tests === 'string') {
-    // It's a filepath, load from CSV
-    const vars = await readVars(tests, basePath);
-    return vars.map((row, idx) => {
-      const test = testCaseFromCsvRow(row);
-      test.description = `Row #${idx + 1}`;
-      return test;
-    });
+    if (tests.endsWith('yaml') || tests.endsWith('yml')) {
+      // Load testcase config from yaml
+      await loadTestsFromGlob(tests);
+    } else {
+      // Legacy load CSV
+      const vars = await readTestsFile(tests, basePath);
+      return vars.map((row, idx) => {
+        const test = testCaseFromCsvRow(row);
+        test.description = `Row #${idx + 1}`;
+        return test;
+      });
+    }
+  } else if (Array.isArray(tests)) {
+    for (const maybeTestsGlob of tests) {
+      if (typeof maybeTestsGlob === 'string') {
+        // Assume it's a filepath
+        await loadTestsFromGlob(maybeTestsGlob);
+      } else {
+        // Assume it's a full test case
+        ret.push(maybeTestsGlob);
+      }
+    }
   }
 
   // Some validation of the shape of tests
-  for (const test of tests) {
+  for (const test of ret) {
     if (!test.assert && !test.vars) {
       throw new Error(
         `Test case must have either "assert" or "vars" property. Instead got ${JSON.stringify(
@@ -276,7 +328,7 @@ export async function readTests(
     }
   }
 
-  return tests;
+  return ret;
 }
 
 export function writeOutput(


Test patch:
diff --git a/examples/separate-test-configs/README.md b/examples/separate-test-configs/README.md
new file mode 100644
index 000000000..4fdf54e91
--- /dev/null
+++ b/examples/separate-test-configs/README.md
@@ -0,0 +1,7 @@
+This example shows how you can use paths and wildcards to simplify for `promptfooconfig.yaml`.
+
+Run:
+
+```
+promptfoo eval
+```
diff --git a/examples/separate-test-configs/promptfooconfig.yaml b/examples/separate-test-configs/promptfooconfig.yaml
new file mode 100644
index 000000000..0b5dfd3cc
--- /dev/null
+++ b/examples/separate-test-configs/promptfooconfig.yaml
@@ -0,0 +1,20 @@
+prompts: prompts.txt
+providers: openai:gpt-3.5-turbo
+
+# The defaultTest configuration is applied to every test case in this config.
+defaultTest:
+  options:
+    suffix: Be extremely concise
+
+# Loads & runs all test cases matching these filepaths
+tests:
+  # You can supply an exact filepath
+  - tests/tests2.yaml
+
+  # Or a glob (wildcard)
+  - tests/*
+
+  # Mix and match with actual test cases
+  - vars:
+      topic: the internet
+      content_type: witty tweets
diff --git a/examples/separate-test-configs/prompts.txt b/examples/separate-test-configs/prompts.txt
new file mode 100644
index 000000000..f8fcb1683
--- /dev/null
+++ b/examples/separate-test-configs/prompts.txt
@@ -0,0 +1,3 @@
+Generate a list of creative {{content_type}} about {{topic}}.
+---
+Imagine you are an expert in {{topic}}. What {{content_type}} would you recommend writing about?
diff --git a/examples/separate-test-configs/tests/tests1.yaml b/examples/separate-test-configs/tests/tests1.yaml
new file mode 100644
index 000000000..07aed53d1
--- /dev/null
+++ b/examples/separate-test-configs/tests/tests1.yaml
@@ -0,0 +1,24 @@
+# This file contains a list of test cases. They're added alongside test cases
+# from tests2.yaml in the example promptfooconfig.yaml.
+
+- vars:
+    topic: artificial intelligence
+    content_type: blog post ideas
+  assert:
+    - type: javascript
+      value: output.length > 100
+- vars:
+    topic: climate change
+    content_type: policy proposals
+- vars:
+    topic: vegetarian cooking
+    content_type: recipe ideas
+- vars:
+    topic: cybersecurity
+    content_type: educational topics
+- vars:
+    topic: remote work
+    content_type: challenges
+- vars:
+    topic: quantum computing
+    content_type: use cases
diff --git a/examples/separate-test-configs/tests/tests2.yaml b/examples/separate-test-configs/tests/tests2.yaml
new file mode 100644
index 000000000..364cb436f
--- /dev/null
+++ b/examples/separate-test-configs/tests/tests2.yaml
@@ -0,0 +1,25 @@
+# This file contains a list of test cases. They're added alongside test cases
+# from tests1.yaml in the example promptfooconfig.yaml.
+
+- vars:
+    topic: alien invasions
+    content_type: survival tips
+  assert:
+    - type: javascript
+      value: output.length > 100
+- vars:
+    topic: time travel
+    content_type: tourist attractions
+- vars:
+    topic: zombie apocalypse
+    content_type: home renovation ideas
+- vars:
+    topic: unicorn breeding
+    content_type: best practices
+- vars:
+    topic: telepathy for beginners
+    content_type: etiquette rules
+- vars:
+    topic: fairy dust
+    content_type: cleaning hacks
+- vars: ../vars/vars_*.yaml
diff --git a/examples/separate-test-configs/vars/vars_extra.yaml b/examples/separate-test-configs/vars/vars_extra.yaml
new file mode 100644
index 000000000..b73272351
--- /dev/null
+++ b/examples/separate-test-configs/vars/vars_extra.yaml
@@ -0,0 +1,2 @@
+topic: bananas
+content_type: musings of an evil genius
diff --git a/test/util.test.ts b/test/util.test.ts
index 43723d113..77b074043 100644
--- a/test/util.test.ts
+++ b/test/util.test.ts
@@ -5,7 +5,7 @@ import yaml from 'js-yaml';
 import { globSync } from 'glob';
 
 import {
-  readVars,
+  readTestsFile,
   readPrompts,
   writeOutput,
   readTests,
@@ -137,7 +137,7 @@ describe('util', () => {
     (fs.readFileSync as jest.Mock).mockReturnValue('var1,var2\nvalue1,value2');
     const varsPath = 'vars.csv';
 
-    const result = await readVars(varsPath);
+    const result = await readTestsFile(varsPath);
 
     expect(fs.readFileSync).toHaveBeenCalledTimes(1);
     expect(result).toEqual([{ var1: 'value1', var2: 'value2' }]);
@@ -147,7 +147,7 @@ describe('util', () => {
     (fs.readFileSync as jest.Mock).mockReturnValue('[{"var1": "value1", "var2": "value2"}]');
     const varsPath = 'vars.json';
 
-    const result = await readVars(varsPath);
+    const result = await readTestsFile(varsPath);
 
     expect(fs.readFileSync).toHaveBeenCalledTimes(1);
     expect(result).toEqual([{ var1: 'value1', var2: 'value2' }]);
@@ -157,7 +157,7 @@ describe('util', () => {
     (fs.readFileSync as jest.Mock).mockReturnValue('- var1: value1\n  var2: value2');
     const varsPath = 'vars.yaml';
 
-    const result = await readVars(varsPath);
+    const result = await readTestsFile(varsPath);
 
     expect(fs.readFileSync).toHaveBeenCalledTimes(1);
     expect(result).toEqual([{ var1: 'value1', var2: 'value2' }]);
@@ -447,4 +447,55 @@ describe('readTests', () => {
 
     expect(result).toEqual(input);
   });
+
+  test('readTests with string array input (paths to test configs)', async () => {
+    const testsPaths = ['test1.yaml', 'test2.yaml'];
+    const test1Content = [
+      {
+        description: 'Test 1',
+        vars: { var1: 'value1', var2: 'value2' },
+        assert: [{ type: 'equals', value: 'value1' }],
+      },
+    ];
+    const test2Content = [
+      {
+        description: 'Test 2',
+        vars: { var1: 'value3', var2: 'value4' },
+        assert: [{ type: 'contains-json', value: 'value3' }],
+      },
+    ];
+    (fs.readFileSync as jest.Mock)
+      .mockReturnValueOnce(yaml.dump(test1Content))
+      .mockReturnValueOnce(yaml.dump(test2Content));
+    (globSync as jest.Mock).mockImplementation((pathOrGlob) => [pathOrGlob]);
+
+    const result = await readTests(testsPaths);
+
+    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
+    expect(result).toEqual([...test1Content, ...test2Content]);
+  });
+
+  test('readTests with vars glob input (paths to vars configs)', async () => {
+    const testsPaths = ['test1.yaml'];
+    const test1Content = [
+      {
+        description: 'Test 1',
+        vars: 'vars1.yaml',
+        assert: [{ type: 'equals', value: 'value1' }],
+      },
+    ];
+    const vars1Content = {
+      var1: 'value1',
+      var2: 'value2',
+    };
+    (fs.readFileSync as jest.Mock)
+      .mockReturnValueOnce(yaml.dump(test1Content))
+      .mockReturnValueOnce(yaml.dump(vars1Content));
+    (globSync as jest.Mock).mockImplementation((pathOrGlob) => [pathOrGlob]);
+
+    const result = await readTests(testsPaths);
+
+    expect(fs.readFileSync).toHaveBeenCalledTimes(2);
+    expect(result).toEqual([Object.assign({}, test1Content[0], {vars: vars1Content})]);
+  });
 });