Repository: promptfoo/promptfoo
Pull request #54 (2023-07-06 04:53:11)

Add ability to view raw prompts in web viewer


Patch:
diff --git a/src/evaluator.ts b/src/evaluator.ts
index a17f19c76..f35942ce9 100644
--- a/src/evaluator.ts
+++ b/src/evaluator.ts
@@ -293,7 +293,7 @@ class Evaluator {
 
     const table: EvaluateTable = {
       head: {
-        prompts: prompts.map((p) => p.display),
+        prompts,
         vars: Array.from(varNames).sort(),
         // TODO(ian): add assertions to table?
       },
@@ -383,6 +383,7 @@ class Evaluator {
           pass: row.success,
           score: row.score,
           text: resultText,
+          prompt: row.prompt.raw,
         };
       },
     );
diff --git a/src/table.ts b/src/table.ts
index f32d44836..9c3d239e5 100644
--- a/src/table.ts
+++ b/src/table.ts
@@ -7,7 +7,7 @@ export function generateTable(summary: EvaluateSummary, tableCellMaxLength = 250
   const head = summary.table.head;
   const headLength = head.prompts.length + head.vars.length;
   const table = new Table({
-    head: [...head.prompts, ...head.vars],
+    head: [...head.prompts.map((prompt) => prompt.display), ...head.vars],
     colWidths: Array(headLength).fill(Math.floor(maxWidth / headLength)),
     wordWrap: true,
     wrapOnWordBoundary: false,
diff --git a/src/types.ts b/src/types.ts
index b6dba8deb..4a0accb70 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -95,11 +95,12 @@ export interface EvaluateTableOutput {
   pass: boolean;
   score: number;
   text: string;
+  prompt: string;
 }
 
 export interface EvaluateTable {
   head: {
-    prompts: string[];
+    prompts: Prompt[];
     vars: string[];
   };
 
diff --git a/src/web/client/src/EvalOutputPromptDialog.tsx b/src/web/client/src/EvalOutputPromptDialog.tsx
new file mode 100644
index 000000000..e000d1fc9
--- /dev/null
+++ b/src/web/client/src/EvalOutputPromptDialog.tsx
@@ -0,0 +1,61 @@
+import { useState, useEffect } from 'react';
+import Button from '@mui/material/Button';
+import Dialog from '@mui/material/Dialog';
+import DialogActions from '@mui/material/DialogActions';
+import DialogContent from '@mui/material/DialogContent';
+import DialogTitle from '@mui/material/DialogTitle';
+import TextareaAutosize from '@mui/base/TextareaAutosize';
+import IconButton from '@mui/material/IconButton';
+import ContentCopyIcon from '@mui/icons-material/ContentCopy';
+import CheckIcon from '@mui/icons-material/Check';
+
+interface EvalOutputPromptDialogProps {
+  open: boolean;
+  onClose: () => void;
+  prompt: string;
+  output?: string;
+}
+
+export default function EvalOutputPromptDialog({
+  open,
+  onClose,
+  prompt,
+  output,
+}: EvalOutputPromptDialogProps) {
+  const [copied, setCopied] = useState(false);
+
+  useEffect(() => {
+    setCopied(false);
+  }, [prompt]);
+
+  const copyToClipboard = async (text: string) => {
+    await navigator.clipboard.writeText(text);
+    setCopied(true);
+  };
+
+  return (
+    <Dialog open={open} onClose={onClose} fullWidth maxWidth="lg">
+      <DialogTitle>Prompt</DialogTitle>
+      <DialogContent>
+        <TextareaAutosize readOnly value={prompt} style={{ width: '100%' }} />
+        <IconButton
+          onClick={() => copyToClipboard(prompt)}
+          style={{ position: 'absolute', right: '10px', top: '10px' }}
+        >
+          {copied ? <CheckIcon /> : <ContentCopyIcon />}
+        </IconButton>
+      </DialogContent>
+      {output && (
+        <>
+          <DialogTitle>Output</DialogTitle>
+          <DialogContent>
+            <TextareaAutosize readOnly value={output} style={{ width: '100%' }} />
+          </DialogContent>
+        </>
+      )}
+      <DialogActions>
+        <Button onClick={onClose}>Close</Button>
+      </DialogActions>
+    </Dialog>
+  );
+}
diff --git a/src/web/client/src/ResultsTable.css b/src/web/client/src/ResultsTable.css
index 3e4378cf5..c86c96906 100644
--- a/src/web/client/src/ResultsTable.css
+++ b/src/web/client/src/ResultsTable.css
@@ -64,10 +64,17 @@ th,
   vertical-align: bottom;
 }
 
+th .action {
+  cursor: pointer;
+  margin-left: 0.5rem;
+}
+
 tr .cell {
 }
 
-tr .cell-rating {
+tr .cell-actions {
+  display: flex;
+  gap: 0.5rem;
   visibility: hidden;
   position: absolute;
   bottom: 1.25rem;
@@ -76,18 +83,14 @@ tr .cell-rating {
   font-size: 1.75rem;
 }
 
-tr:hover .cell-rating {
+tr:hover .cell-actions {
   visibility: visible;
 }
 
-tr .cell-rating .rating {
+tr .cell-actions .action {
   cursor: pointer;
 }
 
-tr .cell-rating .rating:first-child {
-  margin-right: 0.5rem;
-}
-
 th .smalltext {
   visibility: hidden;
   font-weight: normal;
diff --git a/src/web/client/src/ResultsTable.tsx b/src/web/client/src/ResultsTable.tsx
index edef5d242..b03a7b57f 100644
--- a/src/web/client/src/ResultsTable.tsx
+++ b/src/web/client/src/ResultsTable.tsx
@@ -16,6 +16,8 @@ import { useStore } from './store.js';
 
 import type { CellContext, VisibilityState } from '@tanstack/table-core';
 
+import EvalOutputPromptDialog from './EvalOutputPromptDialog';
+
 import type { EvalRow, EvalRowOutput, FilterMode } from './types.js';
 
 import './ResultsTable.css';
@@ -62,28 +64,24 @@ function TruncatedText({ text: rawText, maxLength }: TruncatedTextProps) {
 
   const renderTruncatedText = () => {
     if (text.length <= maxLength) {
-      return text;
+      return <span>text</span>;
     }
     if (isTruncated) {
       return (
-        <>
-          <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
-            {text.substring(0, maxLength)} ...
-          </span>
-        </>
+        <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
+          {text.substring(0, maxLength)} ...
+        </span>
       );
     } else {
       return (
-        <>
-          <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
-            {text}
-          </span>
-        </>
+        <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
+          {text}
+        </span>
       );
     }
   };
 
-  return <div>{renderTruncatedText()}</div>;
+  return renderTruncatedText();
 }
 
 interface PromptOutputProps {
@@ -101,6 +99,13 @@ function EvalOutputCell({
   promptIndex,
   onRating,
 }: PromptOutputProps) {
+  const [openPrompt, setOpen] = React.useState(false);
+  const handlePromptOpen = () => {
+    setOpen(true);
+  };
+  const handlePromptClose = () => {
+    setOpen(false);
+  };
   let text = typeof output.text === 'string' ? output.text : JSON.stringify(output.text);
   let chunks: string[] = [];
   if (!output.pass && text.includes('---')) {
@@ -113,6 +118,7 @@ function EvalOutputCell({
     onRating(rowIndex, promptIndex, isPass);
   };
 
+  // TODO(ian): output.prompt check for backwards compatibility, remove after 0.17.0
   return (
     <>
       <div className="cell">
@@ -128,11 +134,24 @@ function EvalOutputCell({
         )}{' '}
         <TruncatedText text={text} maxLength={maxTextLength} />
       </div>
-      <div className="cell-rating">
-        <span className="rating" onClick={() => handleClick(true)}>
+      <div className="cell-actions">
+        {output.prompt && (
+          <>
+            <span className="action" onClick={handlePromptOpen}>
+              🔎
+            </span>
+            <EvalOutputPromptDialog
+              open={openPrompt}
+              onClose={handlePromptClose}
+              prompt={output.prompt}
+              output={text}
+            />
+          </>
+        )}
+        <span className="action" onClick={() => handleClick(true)}>
           👍
         </span>
-        <span className="rating" onClick={() => handleClick(false)}>
+        <span className="action" onClick={() => handleClick(false)}>
           👎
         </span>
       </div>
@@ -140,11 +159,35 @@ function EvalOutputCell({
   );
 }
 
-function TableHeader({ text, maxLength, smallText }: TruncatedTextProps & { smallText: string }) {
+function TableHeader({
+  text,
+  maxLength,
+  smallText,
+  expandedText,
+}: TruncatedTextProps & { smallText: string; expandedText?: string }) {
+  const [openPrompt, setOpen] = React.useState(false);
+  const handlePromptOpen = () => {
+    setOpen(true);
+  };
+  const handlePromptClose = () => {
+    setOpen(false);
+  };
   return (
     <div>
       <TruncatedText text={text} maxLength={maxLength} />
-      <span className="smalltext">{smallText}</span>
+      {expandedText && (
+        <>
+          <span className="action" onClick={handlePromptOpen}>
+            🔎
+          </span>
+          <EvalOutputPromptDialog
+            open={openPrompt}
+            onClose={handlePromptClose}
+            prompt={expandedText}
+          />
+        </>
+      )}
+      <div className="smalltext">{smallText}</div>
     </div>
   );
 }
@@ -233,11 +276,13 @@ export default function ResultsTable({
               numGood[idx] === highestPassingCount && highestPassingCount !== 0;
             const columnId = `Prompt ${idx + 1}`;
             const isChecked = failureFilter[columnId] || false;
+            // TODO(ian): prompt string support for backwards compatibility, remove after 0.17.0
             return (
               <>
                 <TableHeader
                   smallText={`Prompt ${idx + 1}`}
-                  text={prompt}
+                  text={typeof prompt === 'string' ? prompt : prompt.display}
+                  expandedText={typeof prompt === 'string' ? undefined : prompt.raw}
                   maxLength={maxTextLength}
                 />
                 {filterMode === 'failures' && (
diff --git a/src/web/client/src/types.ts b/src/web/client/src/types.ts
index 61ef2b19e..597a3c6e9 100644
--- a/src/web/client/src/types.ts
+++ b/src/web/client/src/types.ts
@@ -1,5 +1,10 @@
+type Prompt = {
+  display: string;
+  raw: string;
+};
+
 export type EvalHead = {
-  prompts: string[];
+  prompts: Prompt[];
   vars: string[];
 };
 
@@ -7,6 +12,7 @@ export type EvalRowOutput = {
   pass: boolean;
   score: number;
   text: string | object;
+  prompt: string;
 };
 
 export type EvalRow = {


Test patch:
diff --git a/test/util.test.ts b/test/util.test.ts
index 0d4255221..2ac7cbd90 100644
--- a/test/util.test.ts
+++ b/test/util.test.ts
@@ -172,9 +172,15 @@ describe('util', () => {
       },
     ];
     const table: EvaluateTable = {
-      head: { prompts: ['Test prompt'], vars: ['var1', 'var2'] },
+      head: {
+        prompts: [{ raw: 'Test prompt', display: '[display] Test prompt' }],
+        vars: ['var1', 'var2'],
+      },
       body: [
-        { outputs: [{ pass: true, score: 1.0, text: 'Test output' }], vars: ['value1', 'value2'] },
+        {
+          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          vars: ['value1', 'value2'],
+        },
       ],
     };
     const summary = {
@@ -221,9 +227,15 @@ describe('util', () => {
       },
     ];
     const table: EvaluateTable = {
-      head: { prompts: ['Test prompt'], vars: ['var1', 'var2'] },
+      head: {
+        prompts: [{ raw: 'Test prompt', display: '[display] Test prompt' }],
+        vars: ['var1', 'var2'],
+      },
       body: [
-        { outputs: [{ pass: true, score: 1.0, text: 'Test output' }], vars: ['value1', 'value2'] },
+        {
+          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          vars: ['value1', 'value2'],
+        },
       ],
     };
     const summary = {
@@ -270,9 +282,15 @@ describe('util', () => {
       },
     ];
     const table: EvaluateTable = {
-      head: { prompts: ['Test prompt'], vars: ['var1', 'var2'] },
+      head: {
+        prompts: [{ raw: 'Test prompt', display: '[display] Test prompt' }],
+        vars: ['var1', 'var2'],
+      },
       body: [
-        { outputs: [{ pass: true, score: 1.0, text: 'Test output' }], vars: ['value1', 'value2'] },
+        {
+          outputs: [{ pass: true, score: 1.0, text: 'Test output', prompt: 'Test prompt' }],
+          vars: ['value1', 'value2'],
+        },
       ],
     };
     const summary = {