Repository: promptfoo/promptfoo
Pull request #89 (2023-07-25 18:14:45)

Promptfoo Theories

Wanted to contribute a new way of configuring tests / scenarios in promptfoo.  Theories will now allow datasets to be associated with specific set of tests allowing the ability to execute many data driven tests without the requirement of copying a test over and over for each data set. 
Related to #73 [Allow separation of 'tests' vs. 'scenarios'](https://github.com/promptfoo/promptfoo/issues/73) 
Example configuration
```
theories:
  - dataSet:
      - vars:
          language: Spanish
          expectedHelloWorld: 'Hola mundo'
      - vars:
          language: French
          expectedHelloWorld: 'Bonjour le monde'
      - vars:
          language: German
          expectedHelloWorld: 'Hallo Welt'

    tests:
      - description: Translated Hello World
        vars:
          input: 'Hello world'
        assert:
        - type: similar
          value: '{{expectedHelloWorld}}'
          threshold: 0.90      
```


typpo: Thanks so much for this contribution.  I'm going to do a pass on the code - perhaps tonight or tomorrow - but looks pretty close to mergeable!

@therealpaulgg curious what you think, would this help out your work?
typpo: Added some tests to the PR.  Will let you have another pass and see if Paul wants to chime in before finalizing 😁 
therealpaulgg: Yup have played with this and this is pretty much exactly what I am looking for!
Skylertodd: Thank you for the formatting and tests.  Glad to hear this aligns well with existing thoughts!  Let me know if there are any other concerns or changes needed. Happy to help make this tool better, I know this simplifies my tasks :) 
typpo: All looks good to me!  One last nitpick.  Would you mind if we named this functionality `scenarios`?  I feel that is less abstract than `theories`.  I'm working on getting the documentation updated, then happy to land this!

Patch:
diff --git a/examples/multiple-translations-theory/README.md b/examples/multiple-translations-theory/README.md
new file mode 100644
index 000000000..f46848751
--- /dev/null
+++ b/examples/multiple-translations-theory/README.md
@@ -0,0 +1,11 @@
+To get started, set your OPENAI_API_KEY environment variable.
+
+Next, change a few of the prompts in prompts.txt and edit promptfooconfig.yaml.
+
+Then run:
+
+```
+promptfoo eval
+```
+
+Afterwards, you can view the results by running `promptfoo view`
diff --git a/examples/multiple-translations-theory/promptfooconfig.yaml b/examples/multiple-translations-theory/promptfooconfig.yaml
new file mode 100644
index 000000000..dec3e8c10
--- /dev/null
+++ b/examples/multiple-translations-theory/promptfooconfig.yaml
@@ -0,0 +1,44 @@
+prompts: prompts.txt
+providers: [openai:gpt-3.5-turbo, openai:gpt-4]
+theories:
+  - dataSet:
+      - vars:
+          language: Spanish
+          expectedHelloWorld: 'Hola mundo'
+          expectedGoodMorning: 'Buenos días'
+          expectedHowAreYou: '¿Cómo estás?'
+      - vars:
+          language: French
+          expectedHelloWorld: 'Bonjour le monde'
+          expectedGoodMorning: 'Bonjour'
+          expectedHowAreYou: 'Comment ça va?'
+      - vars:
+          language: German
+          expectedHelloWorld: 'Hallo Welt'
+          expectedGoodMorning: 'Guten Morgen'
+          expectedHowAreYou: 'Wie geht es dir?'
+
+    tests:
+      - description: Translated Hello World
+        vars:
+          input: 'Hello world'
+        assert:
+          - type: similar
+            value: '{{expectedHelloWorld}}'
+            threshold: 0.90
+
+      - description: Translated Good Morning
+        vars:
+          input: 'Good morning'
+        assert:
+          - type: similar
+            value: '{{expectedGoodMorning}}'
+            threshold: 0.90
+
+      - description: Translated How are you?
+        vars:
+          input: 'How are you?'
+        assert:
+          - type: similar
+            value: '{{expectedHowAreYou}}'
+            threshold: 0.90
diff --git a/examples/multiple-translations-theory/prompts.txt b/examples/multiple-translations-theory/prompts.txt
new file mode 100644
index 000000000..5c459aab8
--- /dev/null
+++ b/examples/multiple-translations-theory/prompts.txt
@@ -0,0 +1,3 @@
+You're a translator.  Translate this into {{language}}: {{input}}
+---
+Speak in {{language}}: {{input}}
diff --git a/src/assertions.ts b/src/assertions.ts
index 261946e88..6e02e79d4 100644
--- a/src/assertions.ts
+++ b/src/assertions.ts
@@ -99,12 +99,21 @@ export async function runAssertion(
     type: baseType,
   });
 
+  //render assertion values
+  let renderedValue = assertion.value;
+  // renderString for assertion values
+  if (renderedValue && typeof renderedValue === 'string') {
+    renderedValue = nunjucks.renderString(renderedValue, test.vars || {});
+  } else if (renderedValue && Array.isArray(renderedValue)) {
+    renderedValue = renderedValue.map((v) => nunjucks.renderString(v, test.vars || {}));
+  }
+
   if (baseType === 'equals') {
-    pass = assertion.value === output;
+    pass = renderedValue === output;
     return {
       pass,
       score: pass ? 1 : 0,
-      reason: pass ? 'Assertion passed' : `Expected output "${assertion.value}"`,
+      reason: pass ? 'Assertion passed' : `Expected output "${renderedValue}"`,
     };
   }
 
@@ -123,103 +132,99 @@ export async function runAssertion(
   }
 
   if (baseType === 'contains') {
-    invariant(assertion.value, '"contains" assertion type must have a string or number value');
+    invariant(renderedValue, '"contains" assertion type must have a string or number value');
     invariant(
-      typeof assertion.value === 'string' || typeof assertion.value === 'number',
+      typeof renderedValue === 'string' || typeof renderedValue === 'number',
       '"contains" assertion type must have a string or number value',
     );
-    pass = output.includes(String(assertion.value)) !== inverse;
+    pass = output.includes(String(renderedValue)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain "${assertion.value}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
     };
   }
 
   if (baseType === 'contains-any') {
-    invariant(assertion.value, '"contains-any" assertion type must have a value');
+    invariant(renderedValue, '"contains-any" assertion type must have a value');
     invariant(
-      Array.isArray(assertion.value),
+      Array.isArray(renderedValue),
       '"contains-any" assertion type must have an array value',
     );
-    pass = assertion.value.some((value) => output.includes(value)) !== inverse;
+    pass = renderedValue.some((value) => output.includes(value)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain one of "${assertion.value.join(
-            ', ',
-          )}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
     };
   }
 
   if (baseType === 'contains-all') {
-    invariant(assertion.value, '"contains-all" assertion type must have a value');
+    invariant(renderedValue, '"contains-all" assertion type must have a value');
     invariant(
-      Array.isArray(assertion.value),
+      Array.isArray(renderedValue),
       '"contains-all" assertion type must have an array value',
     );
-    pass = assertion.value.every((value) => output.includes(value)) !== inverse;
+    pass = renderedValue.every((value) => output.includes(value)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain all of "${assertion.value.join(
-            ', ',
-          )}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}contain all of "${renderedValue.join(', ')}"`,
     };
   }
 
   if (baseType === 'regex') {
-    invariant(assertion.value, '"regex" assertion type must have a string value');
+    invariant(renderedValue, '"regex" assertion type must have a string value');
     invariant(
-      typeof assertion.value === 'string',
+      typeof renderedValue === 'string',
       '"contains" assertion type must have a string value',
     );
-    const regex = new RegExp(assertion.value);
+    const regex = new RegExp(renderedValue);
     pass = regex.test(output) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}match regex "${assertion.value}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}match regex "${renderedValue}"`,
     };
   }
 
   if (baseType === 'icontains') {
-    invariant(assertion.value, '"icontains" assertion type must have a string or number value');
+    invariant(renderedValue, '"icontains" assertion type must have a string or number value');
     invariant(
-      typeof assertion.value === 'string' || typeof assertion.value === 'number',
+      typeof renderedValue === 'string' || typeof renderedValue === 'number',
       '"icontains" assertion type must have a string or number value',
     );
-    pass = output.toLowerCase().includes(String(assertion.value).toLowerCase()) !== inverse;
+    pass = output.toLowerCase().includes(String(renderedValue).toLowerCase()) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain "${assertion.value}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
     };
   }
 
   if (baseType === 'starts-with') {
-    invariant(assertion.value, '"starts-with" assertion type must have a string value');
+    invariant(renderedValue, '"starts-with" assertion type must have a string value');
     invariant(
-      typeof assertion.value === 'string',
+      typeof renderedValue === 'string',
       '"starts-with" assertion type must have a string value',
     );
-    pass = output.startsWith(String(assertion.value)) !== inverse;
+    pass = output.startsWith(String(renderedValue)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}start with "${assertion.value}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}start with "${renderedValue}"`,
     };
   }
 
@@ -243,7 +248,7 @@ export async function runAssertion(
       if (typeof assertion.value === 'function') {
         return assertion.value(output, test, assertion);
       }
-      const customFunction = new Function('output', 'context', `return ${assertion.value}`);
+      const customFunction = new Function('output', 'context', `return ${renderedValue}`);
       const result = customFunction(output, context) as any;
       if (typeof result === 'boolean') {
         pass = result !== inverse;
@@ -259,7 +264,7 @@ export async function runAssertion(
         pass: false,
         score: 0,
         reason: `Custom function threw error: ${(err as Error).message}
-${assertion.value}`,
+${renderedValue}`,
       };
     }
     return {
@@ -268,7 +273,7 @@ ${assertion.value}`,
       reason: pass
         ? 'Assertion passed'
         : `Custom function returned ${inverse ? 'true' : 'false'}
-${assertion.value}`,
+${renderedValue}`,
     };
   }
 
@@ -311,36 +316,33 @@ ${assertion.value}`,
   }
 
   if (baseType === 'similar') {
-    invariant(assertion.value, 'Similarity assertion must have a string value');
+    invariant(renderedValue, 'Similarity assertion must have a string value');
     invariant(
-      typeof assertion.value === 'string',
+      typeof renderedValue === 'string',
       '"contains" assertion type must have a string value',
     );
-    return matchesSimilarity(assertion.value, output, assertion.threshold || 0.75, inverse);
+    return matchesSimilarity(renderedValue, output, assertion.threshold || 0.75, inverse);
   }
 
   if (baseType === 'llm-rubric') {
-    invariant(assertion.value, 'Similarity assertion must have a string value');
+    invariant(renderedValue, 'Similarity assertion must have a string value');
     invariant(
-      typeof assertion.value === 'string',
+      typeof renderedValue === 'string',
       '"contains" assertion type must have a string value',
     );
-    return matchesLlmRubric(assertion.value, output, test.options);
+    return matchesLlmRubric(renderedValue, output, test.options);
   }
 
   if (baseType === 'webhook') {
-    invariant(assertion.value, '"webhook" assertion type must have a URL value');
-    invariant(
-      typeof assertion.value === 'string',
-      '"webhook" assertion type must have a URL value',
-    );
+    invariant(renderedValue, '"webhook" assertion type must have a URL value');
+    invariant(typeof renderedValue === 'string', '"webhook" assertion type must have a URL value');
 
     try {
       const context = {
         vars: test.vars || {},
       };
       const response = await fetchWithRetries(
-        assertion.value,
+        renderedValue,
         {
           method: 'POST',
           headers: {
@@ -382,10 +384,10 @@ ${assertion.value}`,
 
   if (baseType === 'rouge-n') {
     invariant(
-      typeof assertion.value === 'string' || Array.isArray(assertion.value),
+      typeof renderedValue === 'string' || Array.isArray(renderedValue),
       '"rouge" assertion type must be a value (string or string array)',
     );
-    return handleRougeScore(baseType, assertion, assertion.value, output, inverse);
+    return handleRougeScore(baseType, assertion, renderedValue, output, inverse);
   }
 
   throw new Error('Unknown assertion type: ' + assertion.type);
diff --git a/src/evaluator.ts b/src/evaluator.ts
index 4413cc819..21dc42e11 100644
--- a/src/evaluator.ts
+++ b/src/evaluator.ts
@@ -255,10 +255,11 @@ class Evaluator {
     }
 
     // Aggregate all vars across test cases
-
-    const tests = (
+    let tests = (
       testSuite.tests && testSuite.tests.length > 0
         ? testSuite.tests
+        : testSuite.theories
+        ? []
         : [
             {
               // Dummy test for cases when we're only comparing raw prompts.
@@ -269,6 +270,35 @@ class Evaluator {
       return Object.assign(finalTestCase, test);
     });
 
+    //build theories and add to tests
+    if (testSuite.theories && testSuite.theories.length > 0) {
+      for (const theory of testSuite.theories) {
+        for (const data of theory.dataSet) {
+          //merge defaultTest with TheoryData
+          const theoryTests = (
+            theory.tests || [
+              {
+                // Dummy test for cases when we're only comparing raw prompts.
+              },
+            ]
+          ).map((test) => {
+            return {
+              ...testSuite.defaultTest,
+              ...data,
+              ...test,
+              vars: {
+                ...testSuite.defaultTest?.vars,
+                ...data.vars,
+                ...test.vars,
+              },
+            };
+          });
+          //add theory tests to tests
+          tests = tests.concat(theoryTests);
+        }
+      }
+    }
+
     const varNames: Set<string> = new Set();
     const varsWithSpecialColsRemoved: Record<string, string | string[] | object>[] = [];
     for (const testCase of tests) {
@@ -352,8 +382,7 @@ class Evaluator {
     // Set up progress bar...
     let progressbar: SingleBar | undefined;
     if (options.showProgressBar) {
-      const totalNumRuns =
-        testSuite.prompts.length * testSuite.providers.length * (totalVarCombinations || 1);
+      const totalNumRuns = runEvalOptions.length;
       const cliProgress = await import('cli-progress');
       progressbar = new cliProgress.SingleBar(
         {
diff --git a/src/main.ts b/src/main.ts
index 6f63f3323..c2b8477e6 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -281,6 +281,7 @@ async function main() {
         prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts,
         providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers,
         tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests,
+        theories: fileConfig.theories,
         sharing:
           process.env.PROMPTFOO_DISABLE_SHARING === '1'
             ? false
@@ -310,6 +311,18 @@ async function main() {
         config.tests,
         cmdObj.tests ? undefined : basePath,
       );
+
+      //parse testCases for each theory
+      if (fileConfig.theories) {
+        for (const theory of fileConfig.theories) {
+          const parsedTheoryTests: TestCase[] = await readTests(
+            theory.tests,
+            cmdObj.tests ? undefined : basePath,
+          );
+          theory.tests = parsedTheoryTests;
+        }
+      }
+
       const parsedProviderPromptMap = readProviderPromptMap(config, parsedPrompts);
 
       if (parsedPrompts.length === 0) {
@@ -334,6 +347,7 @@ async function main() {
         providers: parsedProviders,
         providerPromptMap: parsedProviderPromptMap,
         tests: parsedTests,
+        theories: config.theories,
         defaultTest,
       };
 
diff --git a/src/table.ts b/src/table.ts
index ce11d4c7b..a767facd3 100644
--- a/src/table.ts
+++ b/src/table.ts
@@ -24,11 +24,11 @@ export function generateTable(summary: EvaluateSummary, tableCellMaxLength = 250
           text = text.slice(0, tableCellMaxLength) + '...';
         }
         if (pass) {
-          return chalk.green.bold('[PASS] ') + text;
+          return chalk.green('[PASS] ') + text;
         } else if (!pass) {
           // color everything red up until '---'
           return (
-            chalk.red.bold('[FAIL] ') +
+            chalk.red('[FAIL] ') +
             text
               .split('---')
               .map((c, idx) => (idx === 0 ? chalk.red.bold(c) : c))
diff --git a/src/types.ts b/src/types.ts
index f9c69a48e..787da0935 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -192,9 +192,6 @@ export interface TestCase {
   // Key-value pairs to substitute in the prompt
   vars?: Record<string, string | string[] | object>;
 
-  // Optional filepath or glob pattern to load vars from
-  loadVars?: string | string[];
-
   // Optional list of automatic checks to run on the LLM output
   assert?: Assertion[];
 
@@ -202,6 +199,17 @@ export interface TestCase {
   options?: PromptConfig & OutputConfig & GradingConfig;
 }
 
+export interface Theory {
+  // Optional description of what you're testing
+  description?: string;
+
+  // Default test case config
+  dataSet: Partial<TestCase>[];
+
+  // Optional list of automatic checks to run on the LLM output
+  tests: TestCase[];
+}
+
 // Same as a TestCase, except the `vars` object has been flattened into its final form.
 export interface AtomicTestCase extends TestCase {
   vars?: Record<string, string | object>;
@@ -225,6 +233,9 @@ export interface TestSuite {
   // Test cases
   tests?: TestCase[];
 
+  // Theories
+  theories?: Theory[];
+
   // Default test case config
   defaultTest?: Partial<TestCase>;
 }
@@ -249,6 +260,9 @@ export interface TestSuiteConfig {
   // Path to a test file, OR list of LLM prompt variations (aka "test case")
   tests: string | string[] | TestCase[];
 
+  // Theories, groupings of data and tests to be evaluated
+  theories?: Theory[];
+
   // Sets the default properties for each test case. Useful for setting an assertion, on all test cases, for example.
   defaultTest?: Omit<TestCase, 'description'>;
 
diff --git a/src/util.ts b/src/util.ts
index 640653673..5a18d1c6e 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -456,7 +456,7 @@ export function writeLatestResults(results: EvaluateSummary, config: Partial<Uni
         2,
       ),
     );
-    if (fs.existsSync(latestResultsPath)) {
+    if (fs.existsSync(latestResultsPath) || fs.lstatSync(latestResultsPath).isSymbolicLink()) {
       fs.unlinkSync(latestResultsPath);
     }
     fs.symlinkSync(newResultsPath, latestResultsPath);
@@ -473,7 +473,7 @@ export function listPreviousResults(): string[] {
   const sortedFiles = resultsFiles.sort((a, b) => {
     const statA = fs.statSync(path.join(directory, a));
     const statB = fs.statSync(path.join(directory, b));
-    return statB.birthtime.getTime() - statA.birthtime.getTime(); // sort in descending order
+    return statA.birthtime.getTime() - statB.birthtime.getTime(); // sort in ascending order
   });
   return sortedFiles;
 }


Test patch:
diff --git a/test/evaluator.test.ts b/test/evaluator.test.ts
index 945b38f4b..c8595ff31 100644
--- a/test/evaluator.test.ts
+++ b/test/evaluator.test.ts
@@ -415,4 +415,121 @@ describe('evaluator', () => {
     expect(summary.results[0].prompt.display).toBe('Test prompt 1');
     expect(summary.results[0].response?.output).toBe('Test output');
   });
+
+  test('evaluate with theories', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: jest.fn().mockReturnValue('test-provider'),
+      callApi: jest
+        .fn()
+        .mockResolvedValueOnce({
+          output: 'Hola mundo',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        })
+        .mockResolvedValueOnce({
+          output: 'Bonjour le monde',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        }),
+    };
+
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ language }}')],
+      theories: [
+        {
+          dataSet: [
+            {
+              vars: {
+                language: 'Spanish',
+                expectedHelloWorld: 'Hola mundo',
+              },
+            },
+            {
+              vars: {
+                language: 'French',
+                expectedHelloWorld: 'Bonjour le monde',
+              },
+            },
+          ],
+          tests: [
+            {
+              assert: [
+                {
+                  type: 'equals',
+                  value: '{{expectedHelloWorld}}',
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+
+    const summary = await evaluate(testSuite, {});
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+    expect(summary.stats.successes).toBe(2);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Hola mundo');
+    expect(summary.results[1].response?.output).toBe('Bonjour le monde');
+  });
+
+  test('evaluate with theories and multiple vars', async () => {
+    const mockApiProvider: ApiProvider = {
+      id: jest.fn().mockReturnValue('test-provider'),
+      callApi: jest
+        .fn()
+        .mockResolvedValueOnce({
+          output: 'Spanish Hola',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        })
+        .mockResolvedValueOnce({
+          output: 'Spanish Bonjour',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        })
+        .mockResolvedValueOnce({
+          output: 'French Hola',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        })
+        .mockResolvedValueOnce({
+          output: 'French Bonjour',
+          tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        }),
+    };
+    const testSuite: TestSuite = {
+      providers: [mockApiProvider],
+      prompts: [toPrompt('Test prompt {{ language }} {{ greeting }}')],
+      theories: [
+        {
+          dataSet: [
+            {
+              vars: {
+                language: ['Spanish', 'French'],
+                greeting: ['Hola', 'Bonjour'],
+              },
+            },
+          ],
+          tests: [
+            {
+              assert: [
+                {
+                  type: 'equals',
+                  value: '{{language}} {{greeting}}',
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    };
+
+    const summary = await evaluate(testSuite, {});
+
+    expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
+    expect(summary.stats.successes).toBe(4);
+    expect(summary.stats.failures).toBe(0);
+    expect(summary.results[0].response?.output).toBe('Spanish Hola');
+    expect(summary.results[1].response?.output).toBe('Spanish Bonjour');
+    expect(summary.results[2].response?.output).toBe('French Hola');
+    expect(summary.results[3].response?.output).toBe('French Bonjour');
+  });
 });