Repository: promptfoo/promptfoo
Pull request #85 (2023-07-22 16:02:24)

Record latency in results
Record latency and expose it on the webview and in other outputs
![image](https://github.com/promptfoo/promptfoo/assets/310310/c6c89116-362e-4762-8fa7-4a19aa907da1)


cloudflare-pages[bot]: ## Deploying with &nbsp;<a href="https://pages.dev"><img alt="Cloudflare Pages" src="https://user-images.githubusercontent.com/23264/106598434-9e719e00-654f-11eb-9e59-6167043cfa01.png" width="16"></a> &nbsp;Cloudflare Pages

<table><tr><td><strong>Latest commit:</strong> </td><td>
<code>41d022f</code>
</td></tr>
<tr><td><strong>Status:</strong></td><td>&nbsp;✅&nbsp; Deploy successful!</td></tr>
<tr><td><strong>Preview URL:</strong></td><td>
<a href='https://4e5ab821.promptfoo-2.pages.dev'>https://4e5ab821.promptfoo-2.pages.dev</a>
</td></tr>
<tr><td><strong>Branch Preview URL:</strong></td><td>
<a href='https://latency.promptfoo-2.pages.dev'>https://latency.promptfoo-2.pages.dev</a>
</td></tr>
</table>

[View logs](https://dash.cloudflare.com/?to=/:account/pages/view/promptfoo-2/4e5ab821-3fcc-4015-8657-0b2b468bf41c)


Patch:
diff --git a/src/evaluator.ts b/src/evaluator.ts
index 26f1b3fd2..4413cc819 100644
--- a/src/evaluator.ts
+++ b/src/evaluator.ts
@@ -111,13 +111,19 @@ class Evaluator {
       vars,
     };
 
+    let latencyMs = 0;
     try {
+      const startTime = Date.now();
       const response = await provider.callApi(renderedPrompt);
+      const endTime = Date.now();
+      latencyMs = endTime - startTime;
+
       const ret: EvaluateResult = {
         ...setup,
         response,
         success: false,
         score: 0,
+        latencyMs,
       };
       if (response.error) {
         ret.error = response.error;
@@ -177,6 +183,7 @@ class Evaluator {
         error: String(err) + '\n\n' + (err as Error).stack,
         success: false,
         score: 0,
+        latencyMs,
       };
     }
   }
@@ -426,6 +433,8 @@ class Evaluator {
           score: row.score,
           text: resultText,
           prompt: row.prompt.raw,
+          latencyMs: row.latencyMs,
+          tokenUsage: row.response?.tokenUsage,
         };
       },
     );
diff --git a/src/types.ts b/src/types.ts
index 396d55b54..d9034e742 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -96,6 +96,7 @@ export interface EvaluateResult {
   error?: string;
   success: boolean;
   score: number;
+  latencyMs: number;
 }
 
 export interface EvaluateTableOutput {
@@ -103,6 +104,8 @@ export interface EvaluateTableOutput {
   score: number;
   text: string;
   prompt: string;
+  latencyMs: number;
+  tokenUsage?: Partial<TokenUsage>;
 }
 
 export interface EvaluateTable {
diff --git a/src/web/client/src/ResultsTable.css b/src/web/client/src/ResultsTable.css
index c86c96906..bbd9821a5 100644
--- a/src/web/client/src/ResultsTable.css
+++ b/src/web/client/src/ResultsTable.css
@@ -83,7 +83,17 @@ tr .cell-actions {
   font-size: 1.75rem;
 }
 
-tr:hover .cell-actions {
+tr .cell-detail {
+  visibility: hidden;
+  position: absolute;
+  bottom: 0.25rem;
+  margin-top: 1rem;
+  font-size: 0.75rem;
+  color: #888;
+}
+
+tr:hover .cell-actions,
+tr:hover .cell-detail {
   visibility: visible;
 }
 
diff --git a/src/web/client/src/ResultsTable.tsx b/src/web/client/src/ResultsTable.tsx
index e737820de..8b5053619 100644
--- a/src/web/client/src/ResultsTable.tsx
+++ b/src/web/client/src/ResultsTable.tsx
@@ -135,6 +135,16 @@ function EvalOutputCell({
         )}{' '}
         <TruncatedText text={text} maxLength={maxTextLength} />
       </div>
+      <div className="cell-detail">
+        {output.tokenUsage?.cached ? (
+          <span>{output.tokenUsage.cached} tokens (cached)</span>
+        ) : (
+          <>
+            {output.tokenUsage?.total && <span>{output.tokenUsage.total} tokens</span>} |{' '}
+            <span>{output.latencyMs} ms</span>
+          </>
+        )}
+      </div>
       <div className="cell-actions">
         {output.prompt && (
           <>
diff --git a/src/web/client/src/types.ts b/src/web/client/src/types.ts
index 64b0ef09e..780f43694 100644
--- a/src/web/client/src/types.ts
+++ b/src/web/client/src/types.ts
@@ -1,3 +1,5 @@
+import { TokenUsage } from '../../../types';
+
 type Prompt = {
   display: string;
   raw: string;
@@ -13,6 +15,8 @@ export type EvalRowOutput = {
   score: number;
   text: string | object;
   prompt: string;
+  latencyMs: number;
+  tokenUsage?: Partial<TokenUsage>;
 };
 
 export type EvalRow = {


Test patch:
diff --git a/test/util.test.ts b/test/util.test.ts
index 2b1fc6783..43723d113 100644
--- a/test/util.test.ts
+++ b/test/util.test.ts
@@ -169,6 +169,7 @@ describe('util', () => {
       {
         success: true,
         score: 1.0,
+        latencyMs: 1000,
         prompt: {
           raw: 'Test prompt',
           display: '[display] Test prompt',
@@ -189,7 +190,9 @@ describe('util', () => {
       },
       body: [
         {
-          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          outputs: [
+            { pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt', latencyMs: 1000 },
+          ],
           vars: ['value1', 'value2'],
         },
       ],
@@ -224,6 +227,7 @@ describe('util', () => {
       {
         success: true,
         score: 1.0,
+        latencyMs: 1000,
         prompt: {
           raw: 'Test prompt',
           display: '[display] Test prompt',
@@ -244,7 +248,9 @@ describe('util', () => {
       },
       body: [
         {
-          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          outputs: [
+            { pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt', latencyMs: 1000 },
+          ],
           vars: ['value1', 'value2'],
         },
       ],
@@ -279,6 +285,7 @@ describe('util', () => {
       {
         success: true,
         score: 1.0,
+        latencyMs: 1000,
         prompt: {
           raw: 'Test prompt',
           display: '[display] Test prompt',
@@ -299,7 +306,9 @@ describe('util', () => {
       },
       body: [
         {
-          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          outputs: [
+            { pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt', latencyMs: 1000 },
+          ],
           vars: ['value1', 'value2'],
         },
       ],