Repository: promptfoo/promptfoo
Pull request #64 (2023-07-13 00:45:57)

Add starts-with assertion type

cloudflare-pages[bot]: ## Deploying with &nbsp;<a href="https://pages.dev"><img alt="Cloudflare Pages" src="https://user-images.githubusercontent.com/23264/106598434-9e719e00-654f-11eb-9e59-6167043cfa01.png" width="16"></a> &nbsp;Cloudflare Pages

<table><tr><td><strong>Latest commit:</strong> </td><td>
<code>0d1d170</code>
</td></tr>
<tr><td><strong>Status:</strong></td><td>&nbsp;✅&nbsp; Deploy successful!</td></tr>
<tr><td><strong>Preview URL:</strong></td><td>
<a href='https://7fce5bc9.promptfoo-2.pages.dev'>https://7fce5bc9.promptfoo-2.pages.dev</a>
</td></tr>
<tr><td><strong>Branch Preview URL:</strong></td><td>
<a href='https://starts-with.promptfoo-2.pages.dev'>https://starts-with.promptfoo-2.pages.dev</a>
</td></tr>
</table>

[View logs](https://dash.cloudflare.com/?to=/:account/pages/view/promptfoo-2/7fce5bc9-70c9-41a6-993f-e7777eadb43d)


Patch:
diff --git a/README.md b/README.md
index 45636ff3e..66b9fc092 100644
--- a/README.md
+++ b/README.md
@@ -96,6 +96,7 @@ See [Test assertions](https://promptfoo.dev/docs/configuration/expected-outputs)
 | `contains`      | output contains substring                                                 |
 | `icontains`     | output contains substring, case insensitive                               |
 | `regex`         | output matches regex                                                      |
+| `starts-with`   | output starts with string                                                 |
 | `contains-any ` | output contains any of the listed substrings                              |
 | `contains-all`  | output contains all list of substrings                                    |
 | `is-json`       | output is valid json                                                      |
diff --git a/package-lock.json b/package-lock.json
index d1f08db46..a070d95f2 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "promptfoo",
-  "version": "0.17.3",
+  "version": "0.17.4",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "promptfoo",
-      "version": "0.17.3",
+      "version": "0.17.4",
       "license": "MIT",
       "dependencies": {
         "@anthropic-ai/sdk": "^0.5.2",
diff --git a/package.json b/package.json
index 7fc13d45e..2c812b7af 100644
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
   "name": "promptfoo",
   "description": "LLM eval & testing toolkit",
   "author": "Ian Webster",
-  "version": "0.17.3",
+  "version": "0.17.4",
   "license": "MIT",
   "type": "commonjs",
   "main": "dist/src/index.js",
diff --git a/src/assertions.ts b/src/assertions.ts
index 56397f002..cb1642103 100644
--- a/src/assertions.ts
+++ b/src/assertions.ts
@@ -207,6 +207,22 @@ export async function runAssertion(
     };
   }
 
+  if (baseType === 'starts-with') {
+    invariant(assertion.value, '"starts-with" assertion type must have a string value');
+    invariant(
+      typeof assertion.value === 'string',
+      '"starts-with" assertion type must have a string value',
+    );
+    pass = output.startsWith(String(assertion.value)) !== inverse;
+    return {
+      pass,
+      score: pass ? 1 : 0,
+      reason: pass
+        ? 'Assertion passed'
+        : `Expected output to ${inverse ? 'not ' : ''}start with "${assertion.value}"`,
+    };
+  }
+
   if (baseType === 'contains-json') {
     pass = containsJSON(output) !== inverse;
     return {
@@ -480,7 +496,7 @@ export function assertionFromString(expected: string): Assertion {
 
   // New options
   const assertionRegex =
-    /^(not-)?(equals|contains-any|contains-all|contains-json|is-json|regex|icontains|contains|webhook|rouge-n|similar)(?::|\((\d+(\.\d+)?)\):)?(.*)$/;
+    /^(not-)?(equals|contains-any|contains-all|contains-json|is-json|regex|icontains|contains|webhook|rouge-n|similar|starts-with)(?::|\((\d+(\.\d+)?)\):)?(.*)$/;
   const regexMatch = expected.match(assertionRegex);
 
   if (regexMatch) {
@@ -497,7 +513,7 @@ export function assertionFromString(expected: string): Assertion {
       return {
         type: fullType as AssertionType,
       };
-    } else if (type === 'rouge-n' || type === 'similar') {
+    } else if (type === 'rouge-n' || type === 'similar' || type === 'starts-with') {
       return {
         type: fullType as AssertionType,
         value,
diff --git a/src/main.ts b/src/main.ts
index 0d2578db9..33a0e81ef 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -323,7 +323,7 @@ async function main() {
 
       telemetry.maybeShowNotice();
 
-      const border = '='.repeat(process.stdout.columns - 10);
+      const border = '='.repeat((process.stdout.columns || 80) - 10);
       logger.info(border);
       if (!cmdObj.write) {
         logger.info(`${chalk.green('✔')} Evaluation complete`);
diff --git a/src/types.ts b/src/types.ts
index 4a0accb70..cb6787ca9 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -136,6 +136,7 @@ type BaseAssertionTypes =
   | 'icontains'
   | 'contains-all'
   | 'contains-any'
+  | 'starts-with'
   | 'regex'
   | 'is-json'
   | 'contains-json'


Test patch:
diff --git a/test/assertions.test.ts b/test/assertions.test.ts
index 7b8a4a11a..3bb15f94c 100644
--- a/test/assertions.test.ts
+++ b/test/assertions.test.ts
@@ -493,6 +493,36 @@ describe('runAssertion', () => {
     expect(result.pass).toBeFalsy();
     expect(result.reason).toBe('ROUGE-N score 0.2 is less than threshold 0.75');
   });
+
+  // Test for starts-with assertion
+  const startsWithAssertion: Assertion = {
+    type: 'starts-with',
+    value: 'Expected',
+  };
+
+  it('should pass when the starts-with assertion passes', async () => {
+    const output = 'Expected output';
+
+    const result: GradingResult = await runAssertion(
+      startsWithAssertion,
+      {} as AtomicTestCase,
+      output,
+    );
+    expect(result.pass).toBeTruthy();
+    expect(result.reason).toBe('Assertion passed');
+  });
+
+  it('should fail when the starts-with assertion fails', async () => {
+    const output = 'Different output';
+
+    const result: GradingResult = await runAssertion(
+      startsWithAssertion,
+      {} as AtomicTestCase,
+      output,
+    );
+    expect(result.pass).toBeFalsy();
+    expect(result.reason).toBe('Expected output to start with "Expected"');
+  });
 });
 
 describe('assertionFromString', () => {
@@ -632,6 +662,14 @@ describe('assertionFromString', () => {
     expect(result.value).toBe('foo');
     expect(result.threshold).toBeCloseTo(0.225);
   });
+
+  it('should create a starts-with assertion', () => {
+    const expected = 'starts-with:Expected';
+
+    const result: Assertion = assertionFromString(expected);
+    expect(result.type).toBe('starts-with');
+    expect(result.value).toBe('Expected');
+  });
 });
 
 describe('matchesSimilarity', () => {