Repository: firebase/genkit
Pull request #2768: feat(cli/evals): Generate metricSummaries
Author: login: ssbushi
Labels: name: js
name: tooling
name: config
Created: 2025-04-15T20:33:26Z | Merged: 2025-04-29T15:25:00Z

Tooling part of https://github.com/firebase/genkit/issues/1642

Checklist (if applicable):
- [x] PR title is following https://www.conventionalcommits.org/en/v1.0.0/
- [x] Tested (manually, unit tested, etc.)



Diff:
diff --git a/genkit-tools/common/package.json b/genkit-tools/common/package.json
index 77c7e4e604..dfa6652617 100644
--- a/genkit-tools/common/package.json
+++ b/genkit-tools/common/package.json
@@ -12,6 +12,8 @@
     "@asteasolutions/zod-to-openapi": "^7.0.0",
     "@trpc/server": "^10.45.2",
     "adm-zip": "^0.5.12",
+    "ajv": "^8.12.0",
+    "ajv-formats": "^3.0.1",
     "axios": "^1.7.7",
     "body-parser": "^1.20.2",
     "chokidar": "^3.5.3",
@@ -26,8 +28,6 @@
     "json-2-csv": "^5.5.1",
     "json-schema": "^0.4.0",
     "terminate": "^2.6.1",
-    "ajv": "^8.12.0",
-    "ajv-formats": "^3.0.1",
     "tsx": "^4.19.2",
     "uuid": "^9.0.1",
     "winston": "^3.11.0",
diff --git a/genkit-tools/common/src/eval/evaluate.ts b/genkit-tools/common/src/eval/evaluate.ts
index 6415d582b4..b2df6abccf 100644
--- a/genkit-tools/common/src/eval/evaluate.ts
+++ b/genkit-tools/common/src/eval/evaluate.ts
@@ -40,7 +40,11 @@ import {
   logger,
   stackTraceSpans,
 } from '../utils';
-import { enrichResultsWithScoring, extractMetricsMetadata } from './parser';
+import {
+  enrichResultsWithScoring,
+  extractMetricSummaries,
+  extractMetricsMetadata,
+} from './parser';
 
 interface InferenceRunState {
   testCaseId: string;
@@ -181,11 +185,13 @@ export async function runEvaluation(params: {
 
   const scoredResults = enrichResultsWithScoring(scores, evalDataset);
   const metadata = extractMetricsMetadata(evaluatorActions);
+  const metricSummaries = extractMetricSummaries(scores);
 
   const evalRun = {
     key: {
       evalRunId,
       createdAt: new Date().toISOString(),
+      metricSummaries,
       ...augments,
     },
     results: scoredResults,
diff --git a/genkit-tools/common/src/eval/parser.ts b/genkit-tools/common/src/eval/parser.ts
index 1a4ba976c0..42a28cf358 100644
--- a/genkit-tools/common/src/eval/parser.ts
+++ b/genkit-tools/common/src/eval/parser.ts
@@ -20,8 +20,14 @@ import { EvalFnResponse, EvalResponse } from '../types/evaluator';
 import {
   EVALUATOR_METADATA_KEY_DEFINITION,
   EVALUATOR_METADATA_KEY_DISPLAY_NAME,
+  countBy,
+  groupBy,
+  meanBy,
 } from '../utils/eval';
 
+/** Maximum allowed unique strings / enums for generating summaries */
+export const MAX_UNIQUE_STRING_DIST = 5;
+
 /**
  * Combines EvalInput with the generated scores to create a storable EvalResult.
  */
@@ -78,3 +84,127 @@ export function extractMetricsMetadata(evaluatorActions: Action[]) {
   }
   return metadata;
 }
+
+export function extractMetricSummaries(
+  /** key: evaluatorRef */
+  scores: Record<string, EvalResponse>
+) {
+  // key: evaluatorRef or evaluatorRef + scoreId (if available)
+  const testCaseCountMap: Record<string, number> = {};
+
+  const entries = Object.entries(scores)
+    .map(([evaluator, responseArray]) => {
+      testCaseCountMap[evaluator] = responseArray.length;
+      return {
+        evaluator,
+        score: responseArray.flatMap((response) =>
+          Array.isArray(response.evaluation)
+            ? response.evaluation
+            : [response.evaluation]
+        ),
+      };
+    })
+    .flatMap((entry) => {
+      const groupedScores = groupBy(entry.score, 'id');
+      const groupedScoresKeys = Object.keys(groupedScores);
+
+      if (
+        groupedScoresKeys.length === 1 &&
+        groupedScoresKeys[0] === 'undefined'
+      ) {
+        // No score-level granularity
+        return entry.score.flatMap((score) => ({
+          evaluator: entry.evaluator,
+          testCaseCount: testCaseCountMap[entry.evaluator] ?? 0,
+          status: score.status,
+          score: score.score,
+          error: score.error,
+        }));
+      } else {
+        return Object.entries(groupedScores).flatMap(([scoreId, scores]) => {
+          if (scoreId === 'undefined') {
+            return scores.map((score) => ({
+              evaluator: entry.evaluator,
+              testCaseCount: testCaseCountMap[entry.evaluator] ?? 0,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          } else {
+            // Duplicate tracking to simplify lookup.
+            testCaseCountMap[entry.evaluator + '/' + scoreId] =
+              testCaseCountMap[entry.evaluator] ?? 0;
+            return scores.map((score) => ({
+              // Synthetic ID to separate different scores
+              evaluator: entry.evaluator + '/' + scoreId,
+              testCaseCount: testCaseCountMap[entry.evaluator] ?? 0,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          }
+        });
+      }
+    });
+
+  const grouped = groupBy(entries, 'evaluator');
+
+  const summaries = Object.entries(grouped).map(([evaluator, items]) => {
+    const definedItems = items.filter(
+      (item) => typeof item.score !== 'undefined'
+    );
+    const scoreUndefinedCount = items.filter(
+      (item) => typeof item.score === 'undefined'
+    ).length;
+    const errorCount = items.filter((item) => item.error !== undefined).length;
+    const statusDistribution = countBy(items, 'status');
+
+    if (definedItems.length > 0) {
+      // At least one score be registered for this
+      const validItem = definedItems[0];
+      const scoreType = typeof validItem.score;
+      if (scoreType === 'number') {
+        return {
+          evaluator,
+          testCaseCount: validItem.testCaseCount,
+          errorCount,
+          scoreUndefinedCount,
+          statusDistribution,
+          averageScore: meanBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'boolean') {
+        return {
+          evaluator,
+          testCaseCount: validItem.testCaseCount,
+          errorCount,
+          scoreUndefinedCount,
+          statusDistribution,
+          scoreDistribution: countBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'string') {
+        // Treat as enum, but limit to 5 by heuristics
+        const scoreDistribution = countBy(definedItems, 'score');
+
+        if (Object.keys(scoreDistribution).length <= MAX_UNIQUE_STRING_DIST) {
+          return {
+            evaluator,
+            testCaseCount: validItem.testCaseCount,
+            errorCount,
+            scoreUndefinedCount,
+            scoreDistribution,
+            statusDistribution,
+          };
+        }
+      }
+    }
+    return {
+      evaluator,
+      testCaseCount: testCaseCountMap[evaluator] ?? 0,
+      errorCount,
+      scoreUndefinedCount,
+      statusDistribution,
+    };
+  });
+
+  return summaries;
+}
diff --git a/genkit-tools/common/src/types/eval.ts b/genkit-tools/common/src/types/eval.ts
index 01b6621442..90602520cc 100644
--- a/genkit-tools/common/src/types/eval.ts
+++ b/genkit-tools/common/src/types/eval.ts
@@ -131,6 +131,12 @@ export const EvalInputDatasetSchema = z.array(EvalInputSchema);
 export type EvalInputDataset = z.infer<typeof EvalInputDatasetSchema>;
 
 const EvalStatusEnumSchema = z.enum(['UNKNOWN', 'PASS', 'FAIL']);
+/** Enum that indicates if an evaluation has passed or failed */
+export enum EvalStatusEnum {
+  UNKNOWN = 'UNKNOWN',
+  PASS = 'PASS',
+  FAIL = 'FAIL',
+}
 
 export const EvalMetricSchema = z.object({
   evaluator: z.string(),
@@ -164,6 +170,7 @@ export const EvalRunKeySchema = z.object({
   evalRunId: z.string(),
   createdAt: z.string(),
   actionConfig: z.any().optional(),
+  metricSummaries: z.array(z.record(z.string(), z.any())).optional(),
 });
 export type EvalRunKey = z.infer<typeof EvalRunKeySchema>;
 export const EvalKeyAugmentsSchema = EvalRunKeySchema.pick({
diff --git a/genkit-tools/common/src/utils/eval.ts b/genkit-tools/common/src/utils/eval.ts
index 4e26bed85d..265fa7bbde 100644
--- a/genkit-tools/common/src/utils/eval.ts
+++ b/genkit-tools/common/src/utils/eval.ts
@@ -363,3 +363,60 @@ export function getModelInput(data: any, modelConfig: any): GenerateRequest {
     }
   }
 }
+
+/**
+ * Helper method to groupBy an array of objects, replaces lodash equivalent.
+ */
+export function groupBy(
+  arr: any[],
+  criteria: ((i: any) => any) | string
+): Record<string, any[]> {
+  return arr.reduce(function (obj, item) {
+    const key =
+      typeof criteria === 'function' ? criteria(item) : item[criteria];
+
+    if (!obj.hasOwnProperty(key)) {
+      obj[key] = [];
+    }
+    obj[key].push(item);
+
+    return obj;
+  }, {});
+}
+
+/**
+ * Helper method to countBy an array of objects, replaces lodash equivalent.
+ */
+export function countBy(
+  arr: any[],
+  criteria: ((i: any) => any) | string
+): Record<string, number> {
+  return arr.reduce((acc, item) => {
+    const key =
+      typeof criteria === 'function' ? criteria(item) : item[criteria];
+    acc[key] = (acc[key] || 0) + 1;
+
+    return acc;
+  }, {});
+}
+
+/**
+ * Helper method to meanBy an array of objects, replaces lodash equivalent.
+ */
+export function meanBy(
+  arr: any[],
+  criteria: ((i: any) => any) | string
+): number | undefined {
+  if (!arr || arr.length === 0) {
+    return undefined;
+  }
+
+  let sum = 0;
+  for (const item of arr) {
+    const value =
+      typeof criteria === 'function' ? criteria(item) : item[criteria];
+    sum += value;
+  }
+
+  return sum / arr.length;
+}
diff --git a/genkit-tools/common/tests/eval/parser_test.ts b/genkit-tools/common/tests/eval/parser_test.ts
index 42f78417fb..e9f7b55146 100644
--- a/genkit-tools/common/tests/eval/parser_test.ts
+++ b/genkit-tools/common/tests/eval/parser_test.ts
@@ -15,9 +15,16 @@
  */
 
 import { describe, expect, it } from '@jest/globals';
-import { enrichResultsWithScoring } from '../../src/eval';
-import { EvalMetricSchema, EvalResult } from '../../src/types/eval';
-import { EvalResponse } from '../../src/types/evaluator';
+import {
+  enrichResultsWithScoring,
+  extractMetricSummaries,
+} from '../../src/eval';
+import {
+  EvalMetricSchema,
+  EvalResult,
+  EvalStatusEnum,
+} from '../../src/types/eval';
+import { EvalFnResponse, EvalResponse } from '../../src/types/evaluator';
 
 describe('parser', () => {
   const evalRunResults: EvalResult[] = [
@@ -113,4 +120,614 @@ describe('parser', () => {
       });
     });
   });
+
+  describe('extractMetricSummaries', () => {
+    const simpleEvalOutput: Record<string, EvalResponse> = {
+      '/evaluator/genkit/context_relevancy': [
+        {
+          testCaseId: 'case1',
+          evaluation: {
+            score: 7,
+          },
+        },
+        {
+          testCaseId: 'case2',
+          evaluation: {
+            score: 10,
+          },
+        },
+        {
+          testCaseId: 'case3',
+          evaluation: {
+            score: 5,
+          },
+        },
+      ],
+    };
+
+    describe('simpler scenarios', () => {
+      it('mean for simple numeric scores', () => {
+        const results = extractMetricSummaries(simpleEvalOutput);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 0,
+          scoreUndefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          // 7 + 10 + 5
+          averageScore: 22.0 / 3,
+        });
+      });
+
+      it('scoreDistribution for simple boolean scores', () => {
+        const booleanScores = mockScores(simpleEvalOutput, [
+          {
+            score: true,
+          },
+          { score: false },
+          { score: true },
+        ]);
+        const results = extractMetricSummaries(booleanScores);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 0,
+          scoreUndefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          // True, False, True
+          scoreDistribution: { true: 2, false: 1 },
+        });
+      });
+
+      it('scoreDistribution for simple string scores (under 5)', () => {
+        const stringScores = mockScores(simpleEvalOutput, [
+          { score: 'TYPE_0' },
+          { score: 'TYPE_1' },
+          { score: 'TYPE_0' },
+        ]);
+        const results = extractMetricSummaries(stringScores);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 0,
+          scoreUndefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          // TYPE_0, TYPE_1, TYPE_0
+          scoreDistribution: { TYPE_0: 2, TYPE_1: 1 },
+        });
+      });
+
+      it('scoreDistribution for simple string scores (over 5)', () => {
+        const extendedSimpleEvalOutput: Record<string, EvalResponse> = {};
+        // 2x the simpleEvalOutput to get 6 samples.
+        extendedSimpleEvalOutput['/evaluator/genkit/context_relevancy'] = Array(
+          2
+        )
+          .fill(simpleEvalOutput['/evaluator/genkit/context_relevancy'])
+          .flat();
+
+        const stringScores = mockScores(extendedSimpleEvalOutput, [
+          { score: 'TYPE_0' },
+          { score: 'TYPE_1' },
+          { score: 'TYPE_2' },
+          { score: 'TYPE_3' },
+          { score: 'TYPE_4' },
+          { score: 'TYPE_5' },
+        ]);
+        const results = extractMetricSummaries(stringScores);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 6,
+          errorCount: 0,
+          scoreUndefinedCount: 0,
+          statusDistribution: { undefined: 6 },
+        });
+      });
+
+      it('status distribution for simple numeric scores', () => {
+        const scores = [
+          {
+            score: 0,
+            status: EvalStatusEnum.PASS,
+          },
+          {
+            score: 1,
+            status: EvalStatusEnum.FAIL,
+          },
+          {
+            score: 2,
+          },
+        ];
+        const withStatus = mockScores(simpleEvalOutput, scores);
+        const results = extractMetricSummaries(withStatus);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 0,
+          scoreUndefinedCount: 0,
+          statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          // avg(0, 1, 2)
+          averageScore: 3.0 / 3,
+        });
+      });
+    });
+
+    describe('edge cases', () => {
+      it('metrics if scores are undefined but status available', () => {
+        const scores = [
+          {
+            status: EvalStatusEnum.PASS,
+          },
+          {
+            status: EvalStatusEnum.FAIL,
+          },
+          {},
+        ];
+        const undefinedScores = mockScores(simpleEvalOutput, scores);
+        const results = extractMetricSummaries(undefinedScores);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 0,
+          scoreUndefinedCount: 3,
+          // PASS, FAIL, undefined
+          statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+        });
+      });
+
+      it('metrics if some scores are undefined with status available', () => {
+        const scores = [
+          {
+            score: 0,
+            status: EvalStatusEnum.PASS,
+          },
+          {
+            score: 1,
+            status: EvalStatusEnum.FAIL,
+          },
+          {},
+        ];
+        const someDefinedScores = mockScores(simpleEvalOutput, scores);
+        const results = extractMetricSummaries(someDefinedScores);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 0,
+          scoreUndefinedCount: 1,
+          // avg(0, 1)
+          averageScore: 1 / 2.0,
+          // PASS, FAIL, undefined
+          statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+        });
+      });
+
+      it('metrics if some scores are undefined, some errors and with status available', () => {
+        const scores = [
+          {
+            score: undefined,
+            error: 'some error',
+            status: EvalStatusEnum.PASS,
+          },
+          {
+            score: 1,
+            status: EvalStatusEnum.FAIL,
+          },
+          { error: 'some error' },
+        ];
+        const someDefinedScores = mockScores(simpleEvalOutput, scores);
+        const results = extractMetricSummaries(someDefinedScores);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 2,
+          scoreUndefinedCount: 2,
+          // avg(1)
+          averageScore: 1.0,
+          // PASS, FAIL, undefined
+          statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+        });
+      });
+    });
+
+    describe('multiple evaluators grouped', () => {
+      const multiOutput: Record<string, EvalResponse> = {
+        '/evaluator/genkit/faithfulness': [
+          {
+            testCaseId: 'case1',
+            evaluation: {
+              score: 7,
+            },
+          },
+          {
+            testCaseId: 'case2',
+            evaluation: {
+              score: 10,
+            },
+          },
+          {
+            testCaseId: 'case3',
+            evaluation: {
+              score: 5,
+            },
+          },
+        ],
+        '/evaluator/genkit/context_relevancy': [
+          {
+            testCaseId: 'case1',
+            evaluation: {
+              score: true,
+            },
+          },
+          {
+            testCaseId: 'case2',
+            evaluation: {
+              score: false,
+            },
+          },
+          {
+            testCaseId: 'case3',
+            evaluation: {
+              score: true,
+            },
+          },
+        ],
+      };
+
+      it('treats each evaluator separately', () => {
+        const results = extractMetricSummaries(multiOutput);
+
+        expect(results).toHaveLength(2);
+        expect(results).toContainEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 0,
+          scoreUndefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          // true, false, true
+          scoreDistribution: { true: 2, false: 1 },
+        });
+        expect(results).toContainEqual({
+          evaluator: '/evaluator/genkit/faithfulness',
+          testCaseCount: 3,
+          errorCount: 0,
+          scoreUndefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          // avg(7, 10, 5)
+          averageScore: 22.0 / 3,
+        });
+      });
+
+      it('treats each evaluator separately, with errors, status, undefined scores', () => {
+        const mockFaithfulness = [
+          {
+            status: EvalStatusEnum.PASS,
+            error: 'some error',
+          },
+          {
+            score: 10,
+            status: EvalStatusEnum.FAIL,
+          },
+          {
+            error: 'some error',
+          },
+        ];
+        const mockContextRel = [
+          {
+            score: 'alpha',
+            status: EvalStatusEnum.PASS,
+          },
+          {
+            status: EvalStatusEnum.FAIL,
+            error: 'some error',
+          },
+          {
+            score: 'gamma',
+          },
+        ];
+        const someDefinedScores = reMapScores(
+          multiOutput,
+          (response, i, evaluator) => {
+            if (evaluator === '/evaluator/genkit/faithfulness') {
+              return {
+                testCaseId: response.testCaseId,
+                evaluation: mockFaithfulness[i],
+              };
+            } else {
+              return {
+                testCaseId: response.testCaseId,
+                evaluation: mockContextRel[i],
+              };
+            }
+          }
+        );
+        const results = extractMetricSummaries(someDefinedScores);
+        expect(results).toHaveLength(2);
+        expect(results).toContainEqual({
+          evaluator: '/evaluator/genkit/faithfulness',
+          testCaseCount: 3,
+          errorCount: 2,
+          scoreUndefinedCount: 2,
+          statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          // avg(10)
+          averageScore: 10.0,
+        });
+        expect(results).toContainEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          testCaseCount: 3,
+          errorCount: 1,
+          scoreUndefinedCount: 1,
+          statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          // alpha, gamma
+          scoreDistribution: { alpha: 1, gamma: 1 },
+        });
+      });
+
+      describe('multi-scores', () => {
+        it('mix of scores', () => {
+          const mockEvaluations = [
+            {
+              score: 1,
+              status: EvalStatusEnum.PASS,
+            },
+            [
+              {
+                score: 1,
+                status: EvalStatusEnum.FAIL,
+              },
+              {
+                score: 2,
+                status: EvalStatusEnum.PASS,
+              },
+            ],
+            {
+              score: undefined,
+            },
+          ];
+          const mixedScores = reMapScores(simpleEvalOutput, (response, i) => ({
+            testCaseId: response.testCaseId,
+            evaluation: mockEvaluations[i],
+          }));
+          const results = extractMetricSummaries(mixedScores);
+
+          expect(results).toHaveLength(1);
+
+          const result = results[0];
+          expect(result).toEqual({
+            evaluator: '/evaluator/genkit/context_relevancy',
+            testCaseCount: 3,
+            errorCount: 0,
+            scoreUndefinedCount: 1,
+            // avg(1, 1, 2)
+            averageScore: 4.0 / 3,
+            // PASS, FAIL, PASS, undefined
+            statusDistribution: { undefined: 1, PASS: 2, FAIL: 1 },
+          });
+        });
+
+        it('scores with IDs', () => {
+          const mockEvaluations = [
+            [
+              {
+                score: 5,
+                id: 'numeric',
+                status: EvalStatusEnum.PASS,
+              },
+              {
+                score: 'YES',
+                id: 'enum',
+                status: EvalStatusEnum.FAIL,
+              },
+            ],
+            [
+              {
+                score: 7,
+                id: 'numeric',
+                status: EvalStatusEnum.FAIL,
+              },
+              {
+                score: 'NO',
+                id: 'enum',
+                status: EvalStatusEnum.PASS,
+              },
+            ],
+            [
+              {
+                score: undefined,
+                id: 'numeric',
+                error: 'somer error',
+              },
+              {
+                score: undefined,
+                id: 'enum',
+              },
+            ],
+          ];
+          const mixedScores = reMapScores(simpleEvalOutput, (response, i) => ({
+            testCaseId: response.testCaseId,
+            evaluation: mockEvaluations[i],
+          }));
+          const results = extractMetricSummaries(mixedScores);
+
+          expect(results).toHaveLength(2);
+          expect(results).toContainEqual({
+            evaluator: '/evaluator/genkit/context_relevancy/numeric',
+            testCaseCount: 3,
+            errorCount: 1,
+            scoreUndefinedCount: 1,
+            // avg(5, 7)
+            averageScore: 12.0 / 2,
+            // PASS, FAIL, undefined
+            statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          });
+          expect(results).toContainEqual({
+            evaluator: '/evaluator/genkit/context_relevancy/enum',
+            testCaseCount: 3,
+            errorCount: 0,
+            scoreUndefinedCount: 1,
+            // YES, NO
+            scoreDistribution: { YES: 1, NO: 1 },
+            // FAIL, PASS, undefined
+            statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          });
+        });
+
+        it('multi-scores with IDs', () => {
+          const mockEvaluations = [
+            [
+              {
+                score: 5,
+                id: 'numeric',
+                status: EvalStatusEnum.PASS,
+              },
+              {
+                score: 'YES',
+                id: 'enum',
+                status: EvalStatusEnum.FAIL,
+              },
+            ],
+            [
+              {
+                score: 7,
+                id: 'numeric',
+                status: EvalStatusEnum.FAIL,
+              },
+              {
+                score: 'NO',
+                id: 'enum',
+                status: EvalStatusEnum.PASS,
+              },
+            ],
+            [
+              {
+                score: undefined,
+                id: 'numeric',
+                error: 'somer error',
+              },
+              {
+                score: undefined,
+                id: 'enum',
+              },
+            ],
+          ];
+          const mixedScores = reMapScores(multiOutput, (response, i) => ({
+            testCaseId: response.testCaseId,
+            evaluation: mockEvaluations[i],
+          }));
+          const results = extractMetricSummaries(mixedScores);
+
+          expect(results).toHaveLength(4);
+          expect(results).toContainEqual({
+            evaluator: '/evaluator/genkit/context_relevancy/numeric',
+            testCaseCount: 3,
+            errorCount: 1,
+            scoreUndefinedCount: 1,
+            // avg(5, 7)
+            averageScore: 12.0 / 2,
+            // PASS, FAIL, undefined
+            statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          });
+          expect(results).toContainEqual({
+            evaluator: '/evaluator/genkit/context_relevancy/enum',
+            testCaseCount: 3,
+            errorCount: 0,
+            scoreUndefinedCount: 1,
+            // YES, NO
+            scoreDistribution: { YES: 1, NO: 1 },
+            // FAIL, PASS, undefined
+            statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          });
+          expect(results).toContainEqual({
+            evaluator: '/evaluator/genkit/faithfulness/numeric',
+            testCaseCount: 3,
+            errorCount: 1,
+            scoreUndefinedCount: 1,
+            // avg(5, 7)
+            averageScore: 12.0 / 2,
+            // PASS, FAIL, undefined
+            statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          });
+          expect(results).toContainEqual({
+            evaluator: '/evaluator/genkit/faithfulness/enum',
+            testCaseCount: 3,
+            errorCount: 0,
+            scoreUndefinedCount: 1,
+            // YES, NO
+            scoreDistribution: { YES: 1, NO: 1 },
+            // FAIL, PASS, undefined
+            statusDistribution: { undefined: 1, PASS: 1, FAIL: 1 },
+          });
+        });
+      });
+    });
+  });
 });
+
+function reMapScores(
+  scoresMap: Record<string, EvalResponse>,
+  fn: (
+    score: EvalFnResponse,
+    index: number,
+    evaluator?: string
+  ) => EvalFnResponse
+): Record<string, EvalResponse> {
+  let remapped: Record<string, EvalResponse> = {};
+
+  for (const [evaluator, scores] of Object.entries(scoresMap)) {
+    remapped[evaluator] = scores.map((score, index) =>
+      fn(score, index, evaluator)
+    );
+  }
+  return remapped;
+}
+
+function mockScores(
+  scoresMap: Record<string, EvalResponse>,
+  mockedScores: any[] | Record<string, any[]>
+): Record<string, EvalResponse> {
+  let remapped: Record<string, EvalResponse> = {};
+
+  for (const [evaluator, scores] of Object.entries(scoresMap)) {
+    remapped[evaluator] = scores.map((score, index) => {
+      const evaluation = Array.isArray(mockedScores)
+        ? { ...mockedScores[index] }
+        : mockedScores[evaluator][index];
+      return {
+        testCaseId: score.testCaseId,
+        evaluation,
+      };
+    });
+  }
+  return remapped;
+}


Reviews:
user: login: shrutip90
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T21:53:59Z
user: login: ssbushi
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T14:39:18Z
user: login: pavelgj
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-19T13:30:53Z
user: login: shrutip90
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T19:19:03Z

Review comments:
diff_hunk: @@ -78,3 +79,111 @@ export function extractMetricsMetadata(evaluatorActions: Action[]) {
   }
   return metadata;
 }
+
+export function extractMetricSummaries(
+  /** key: evaluatorRef */
+  scores: Record<string, EvalResponse>
+) {
+  const entries = Object.entries(scores)
+    .map(([evaluator, responseArray]) => ({
+      evaluator,
+      score: responseArray.flatMap((response) =>
+        Array.isArray(response.evaluation)
+          ? response.evaluation
+          : [response.evaluation]
+      ),
+    }))
+    .flatMap((entry) => {
+      const groupedScores = _.groupBy(entry.score, 'id');
+      const groupedScoresKeys = Object.keys(groupedScores);
+
+      if (
+        groupedScoresKeys.length === 1 &&
+        groupedScoresKeys[0] === 'undefined'
+      ) {
+        // No score-level granularity
+        return _.flatMap(entry.score, (score) => ({
+          evaluator: entry.evaluator,
+          status: score.status,
+          score: score.score,
+          error: score.error,
+        }));
+      } else {
+        return _.flatMap(groupedScores, (scores, scoreId) => {
+          if (scoreId === 'undefined') {
+            return scores.map((score) => ({
+              evaluator: entry.evaluator,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          } else {
+            return scores.map((score) => ({
+              // Synthetic ID to separate different scores
+              evaluator: entry.evaluator + '/' + scoreId,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          }
+        });
+      }
+    });
+
+  const grouped = _.groupBy(entries, 'evaluator');
+
+  const summaries = _.map(grouped, (items, evaluator) => {
+    const definedItems = items.filter(
+      (item) => typeof item.score !== 'undefined'
+    );
+    const undefinedCount = items.filter(
+      (item) => typeof item.score === 'undefined'
+    ).length;
+    const errorCount = items.filter((item) => item.error !== undefined).length;
+    const statusDistribution = _.countBy(items, 'status');
+
+    if (definedItems.length > 0) {
+      // At least one score be registered for this
+      const validItem = definedItems[0];
+      const scoreType = typeof validItem.score;
+      if (scoreType === 'number') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          average: _.meanBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'boolean') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          distribution: _.countBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'string') {
+        // Treat as enum, but limit to 5 by heuristics
+        const distribution = _.countBy(definedItems, 'score');
+
+        if (Object.keys(distribution).length <= 5) {
+          return {
+            evaluator,
+            errorCount,
+            undefinedCount,
+            distribution,
+            statusDistribution,
+          };
+        }
+      }
+    }
+    return {
+      evaluator,
+      errorCount,
+      undefinedCount,
path: genkit-tools/common/src/eval/parser.ts
user: login: shrutip90
body: This naming is not clear. scoreUnavailableCount or scoreUndefinedCount? (since status could be undefined too and this wont count those cases).
created_at: 2025-04-16T20:45:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -113,4 +120,553 @@ describe('parser', () => {
       });
     });
   });
+
+  describe('extractMetricSummaries', () => {
+    const simpleEvalOutput: Record<string, EvalResponse> = {
+      '/evaluator/genkit/context_relevancy': [
+        {
+          testCaseId: 'case1',
+          evaluation: {
+            score: 7,
+          },
+        },
+        {
+          testCaseId: 'case2',
+          evaluation: {
+            score: 10,
+          },
+        },
+        {
+          testCaseId: 'case3',
+          evaluation: {
+            score: 5,
+          },
+        },
+      ],
+    };
+
+    describe('simpler scenarios', () => {
+      it('mean for simple numeric scores', () => {
+        const results = extractMetricSummaries(simpleEvalOutput);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          errorCount: 0,
+          undefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          average: 22.0 / 3,
+        });
+      });
+
+      it('distribution for simple boolean scores', () => {
+        const booleanScores = reMapScores(simpleEvalOutput, (response, i) => ({
+          testCaseId: response.testCaseId,
+          evaluation: {
+            score: i % 2 === 0,
path: genkit-tools/common/tests/eval/parser_test.ts
user: login: shrutip90
body: This is hard to read. I will have to look at simpleEvalOutput and do the math to understand what this should be. Please just define the test case here.
created_at: 2025-04-16T20:49:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -78,3 +79,111 @@ export function extractMetricsMetadata(evaluatorActions: Action[]) {
   }
   return metadata;
 }
+
+export function extractMetricSummaries(
+  /** key: evaluatorRef */
+  scores: Record<string, EvalResponse>
+) {
+  const entries = Object.entries(scores)
+    .map(([evaluator, responseArray]) => ({
+      evaluator,
+      score: responseArray.flatMap((response) =>
+        Array.isArray(response.evaluation)
+          ? response.evaluation
+          : [response.evaluation]
+      ),
+    }))
+    .flatMap((entry) => {
+      const groupedScores = _.groupBy(entry.score, 'id');
+      const groupedScoresKeys = Object.keys(groupedScores);
+
+      if (
+        groupedScoresKeys.length === 1 &&
+        groupedScoresKeys[0] === 'undefined'
+      ) {
+        // No score-level granularity
+        return _.flatMap(entry.score, (score) => ({
+          evaluator: entry.evaluator,
+          status: score.status,
+          score: score.score,
+          error: score.error,
+        }));
+      } else {
+        return _.flatMap(groupedScores, (scores, scoreId) => {
+          if (scoreId === 'undefined') {
+            return scores.map((score) => ({
+              evaluator: entry.evaluator,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          } else {
+            return scores.map((score) => ({
+              // Synthetic ID to separate different scores
+              evaluator: entry.evaluator + '/' + scoreId,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          }
+        });
+      }
+    });
+
+  const grouped = _.groupBy(entries, 'evaluator');
+
+  const summaries = _.map(grouped, (items, evaluator) => {
+    const definedItems = items.filter(
+      (item) => typeof item.score !== 'undefined'
+    );
+    const undefinedCount = items.filter(
+      (item) => typeof item.score === 'undefined'
+    ).length;
+    const errorCount = items.filter((item) => item.error !== undefined).length;
+    const statusDistribution = _.countBy(items, 'status');
+
+    if (definedItems.length > 0) {
+      // At least one score be registered for this
+      const validItem = definedItems[0];
+      const scoreType = typeof validItem.score;
+      if (scoreType === 'number') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          average: _.meanBy(definedItems, 'score'),
path: genkit-tools/common/src/eval/parser.ts
user: login: shrutip90
body: -> averageScore
created_at: 2025-04-16T20:52:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -78,3 +79,111 @@ export function extractMetricsMetadata(evaluatorActions: Action[]) {
   }
   return metadata;
 }
+
+export function extractMetricSummaries(
+  /** key: evaluatorRef */
+  scores: Record<string, EvalResponse>
+) {
+  const entries = Object.entries(scores)
+    .map(([evaluator, responseArray]) => ({
+      evaluator,
+      score: responseArray.flatMap((response) =>
+        Array.isArray(response.evaluation)
+          ? response.evaluation
+          : [response.evaluation]
+      ),
+    }))
+    .flatMap((entry) => {
+      const groupedScores = _.groupBy(entry.score, 'id');
+      const groupedScoresKeys = Object.keys(groupedScores);
+
+      if (
+        groupedScoresKeys.length === 1 &&
+        groupedScoresKeys[0] === 'undefined'
+      ) {
+        // No score-level granularity
+        return _.flatMap(entry.score, (score) => ({
+          evaluator: entry.evaluator,
+          status: score.status,
+          score: score.score,
+          error: score.error,
+        }));
+      } else {
+        return _.flatMap(groupedScores, (scores, scoreId) => {
+          if (scoreId === 'undefined') {
+            return scores.map((score) => ({
+              evaluator: entry.evaluator,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          } else {
+            return scores.map((score) => ({
+              // Synthetic ID to separate different scores
+              evaluator: entry.evaluator + '/' + scoreId,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          }
+        });
+      }
+    });
+
+  const grouped = _.groupBy(entries, 'evaluator');
+
+  const summaries = _.map(grouped, (items, evaluator) => {
+    const definedItems = items.filter(
+      (item) => typeof item.score !== 'undefined'
+    );
+    const undefinedCount = items.filter(
+      (item) => typeof item.score === 'undefined'
+    ).length;
+    const errorCount = items.filter((item) => item.error !== undefined).length;
+    const statusDistribution = _.countBy(items, 'status');
+
+    if (definedItems.length > 0) {
+      // At least one score be registered for this
+      const validItem = definedItems[0];
+      const scoreType = typeof validItem.score;
+      if (scoreType === 'number') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          average: _.meanBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'boolean') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          distribution: _.countBy(definedItems, 'score'),
path: genkit-tools/common/src/eval/parser.ts
user: login: shrutip90
body: -> scoreDistribution to differentiate from statusDistribution
created_at: 2025-04-16T20:52:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -78,3 +79,111 @@ export function extractMetricsMetadata(evaluatorActions: Action[]) {
   }
   return metadata;
 }
+
+export function extractMetricSummaries(
+  /** key: evaluatorRef */
+  scores: Record<string, EvalResponse>
+) {
+  const entries = Object.entries(scores)
+    .map(([evaluator, responseArray]) => ({
+      evaluator,
+      score: responseArray.flatMap((response) =>
+        Array.isArray(response.evaluation)
+          ? response.evaluation
+          : [response.evaluation]
+      ),
+    }))
+    .flatMap((entry) => {
+      const groupedScores = _.groupBy(entry.score, 'id');
+      const groupedScoresKeys = Object.keys(groupedScores);
+
+      if (
+        groupedScoresKeys.length === 1 &&
+        groupedScoresKeys[0] === 'undefined'
+      ) {
+        // No score-level granularity
+        return _.flatMap(entry.score, (score) => ({
+          evaluator: entry.evaluator,
+          status: score.status,
+          score: score.score,
+          error: score.error,
+        }));
+      } else {
+        return _.flatMap(groupedScores, (scores, scoreId) => {
+          if (scoreId === 'undefined') {
+            return scores.map((score) => ({
+              evaluator: entry.evaluator,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          } else {
+            return scores.map((score) => ({
+              // Synthetic ID to separate different scores
+              evaluator: entry.evaluator + '/' + scoreId,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          }
+        });
+      }
+    });
+
+  const grouped = _.groupBy(entries, 'evaluator');
+
+  const summaries = _.map(grouped, (items, evaluator) => {
+    const definedItems = items.filter(
+      (item) => typeof item.score !== 'undefined'
+    );
+    const undefinedCount = items.filter(
+      (item) => typeof item.score === 'undefined'
+    ).length;
+    const errorCount = items.filter((item) => item.error !== undefined).length;
+    const statusDistribution = _.countBy(items, 'status');
+
+    if (definedItems.length > 0) {
+      // At least one score be registered for this
+      const validItem = definedItems[0];
+      const scoreType = typeof validItem.score;
+      if (scoreType === 'number') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          average: _.meanBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'boolean') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          distribution: _.countBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'string') {
+        // Treat as enum, but limit to 5 by heuristics
+        const distribution = _.countBy(definedItems, 'score');
+
+        if (Object.keys(distribution).length <= 5) {
+          return {
+            evaluator,
+            errorCount,
+            undefinedCount,
+            distribution,
path: genkit-tools/common/src/eval/parser.ts
user: login: shrutip90
body: -> scoreDistribution
created_at: 2025-04-16T20:53:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -113,4 +120,553 @@ describe('parser', () => {
       });
     });
   });
+
+  describe('extractMetricSummaries', () => {
+    const simpleEvalOutput: Record<string, EvalResponse> = {
+      '/evaluator/genkit/context_relevancy': [
+        {
+          testCaseId: 'case1',
+          evaluation: {
+            score: 7,
+          },
+        },
+        {
+          testCaseId: 'case2',
+          evaluation: {
+            score: 10,
+          },
+        },
+        {
+          testCaseId: 'case3',
+          evaluation: {
+            score: 5,
+          },
+        },
+      ],
+    };
+
+    describe('simpler scenarios', () => {
+      it('mean for simple numeric scores', () => {
+        const results = extractMetricSummaries(simpleEvalOutput);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          errorCount: 0,
+          undefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          average: 22.0 / 3,
+        });
+      });
+
+      it('distribution for simple boolean scores', () => {
+        const booleanScores = reMapScores(simpleEvalOutput, (response, i) => ({
+          testCaseId: response.testCaseId,
+          evaluation: {
+            score: i % 2 === 0,
+          },
+        }));
+        const results = extractMetricSummaries(booleanScores);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          errorCount: 0,
+          undefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          distribution: { true: 2, false: 1 },
+        });
+      });
+
+      it('distribution for simple string scores (under 5)', () => {
+        const stringScores = reMapScores(simpleEvalOutput, (response, i) => ({
+          testCaseId: response.testCaseId,
+          evaluation: {
+            score: `TYPE_${i % 2}`,
path: genkit-tools/common/tests/eval/parser_test.ts
user: login: shrutip90
body: ditto. Maybe instead of reMapScores you can have a helper that takes an array of Score objects and generates Record<string, EvalResponse> with testcaseIds. That might be concise enough and easier to read.
created_at: 2025-04-16T20:53:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -78,3 +79,111 @@ export function extractMetricsMetadata(evaluatorActions: Action[]) {
   }
   return metadata;
 }
+
+export function extractMetricSummaries(
+  /** key: evaluatorRef */
+  scores: Record<string, EvalResponse>
+) {
+  const entries = Object.entries(scores)
+    .map(([evaluator, responseArray]) => ({
+      evaluator,
+      score: responseArray.flatMap((response) =>
+        Array.isArray(response.evaluation)
+          ? response.evaluation
+          : [response.evaluation]
+      ),
+    }))
+    .flatMap((entry) => {
+      const groupedScores = _.groupBy(entry.score, 'id');
+      const groupedScoresKeys = Object.keys(groupedScores);
+
+      if (
+        groupedScoresKeys.length === 1 &&
+        groupedScoresKeys[0] === 'undefined'
+      ) {
+        // No score-level granularity
+        return _.flatMap(entry.score, (score) => ({
+          evaluator: entry.evaluator,
+          status: score.status,
+          score: score.score,
+          error: score.error,
+        }));
+      } else {
+        return _.flatMap(groupedScores, (scores, scoreId) => {
+          if (scoreId === 'undefined') {
+            return scores.map((score) => ({
+              evaluator: entry.evaluator,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          } else {
+            return scores.map((score) => ({
+              // Synthetic ID to separate different scores
+              evaluator: entry.evaluator + '/' + scoreId,
+              status: score.status,
+              score: score.score,
+              error: score.error,
+            }));
+          }
+        });
+      }
+    });
+
+  const grouped = _.groupBy(entries, 'evaluator');
+
+  const summaries = _.map(grouped, (items, evaluator) => {
+    const definedItems = items.filter(
+      (item) => typeof item.score !== 'undefined'
+    );
+    const undefinedCount = items.filter(
+      (item) => typeof item.score === 'undefined'
+    ).length;
+    const errorCount = items.filter((item) => item.error !== undefined).length;
+    const statusDistribution = _.countBy(items, 'status');
+
+    if (definedItems.length > 0) {
+      // At least one score be registered for this
+      const validItem = definedItems[0];
+      const scoreType = typeof validItem.score;
+      if (scoreType === 'number') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          average: _.meanBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'boolean') {
+        return {
+          evaluator,
+          errorCount,
+          undefinedCount,
+          statusDistribution,
+          distribution: _.countBy(definedItems, 'score'),
+        };
+      } else if (scoreType === 'string') {
+        // Treat as enum, but limit to 5 by heuristics
+        const distribution = _.countBy(definedItems, 'score');
+
+        if (Object.keys(distribution).length <= 5) {
+          return {
+            evaluator,
+            errorCount,
+            undefinedCount,
+            distribution,
+            statusDistribution,
+          };
+        }
+      }
+    }
+    return {
+      evaluator,
+      errorCount,
path: genkit-tools/common/src/eval/parser.ts
user: login: shrutip90
body: Also add a testCaseCount?
created_at: 2025-04-16T21:03:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -113,4 +120,553 @@ describe('parser', () => {
       });
     });
   });
+
+  describe('extractMetricSummaries', () => {
+    const simpleEvalOutput: Record<string, EvalResponse> = {
+      '/evaluator/genkit/context_relevancy': [
+        {
+          testCaseId: 'case1',
+          evaluation: {
+            score: 7,
+          },
+        },
+        {
+          testCaseId: 'case2',
+          evaluation: {
+            score: 10,
+          },
+        },
+        {
+          testCaseId: 'case3',
+          evaluation: {
+            score: 5,
+          },
+        },
+      ],
+    };
+
+    describe('simpler scenarios', () => {
+      it('mean for simple numeric scores', () => {
+        const results = extractMetricSummaries(simpleEvalOutput);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          errorCount: 0,
+          undefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          average: 22.0 / 3,
+        });
+      });
+
+      it('distribution for simple boolean scores', () => {
path: genkit-tools/common/tests/eval/parser_test.ts
user: login: shrutip90
body: distribution -> score distribution.

Same for other testcases below.
created_at: 2025-04-16T21:06:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -113,4 +120,553 @@ describe('parser', () => {
       });
     });
   });
+
+  describe('extractMetricSummaries', () => {
+    const simpleEvalOutput: Record<string, EvalResponse> = {
+      '/evaluator/genkit/context_relevancy': [
+        {
+          testCaseId: 'case1',
+          evaluation: {
+            score: 7,
+          },
+        },
+        {
+          testCaseId: 'case2',
+          evaluation: {
+            score: 10,
+          },
+        },
+        {
+          testCaseId: 'case3',
+          evaluation: {
+            score: 5,
+          },
+        },
+      ],
+    };
+
+    describe('simpler scenarios', () => {
+      it('mean for simple numeric scores', () => {
+        const results = extractMetricSummaries(simpleEvalOutput);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          errorCount: 0,
+          undefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          average: 22.0 / 3,
+        });
+      });
+
+      it('distribution for simple boolean scores', () => {
+        const booleanScores = reMapScores(simpleEvalOutput, (response, i) => ({
+          testCaseId: response.testCaseId,
+          evaluation: {
+            score: i % 2 === 0,
path: genkit-tools/common/tests/eval/parser_test.ts
user: login: ssbushi
body: I was trying to avoid redefining the EvalResponse object for every test case since it is already a large object, I can add comments to the override function here to make the scores more obvious. LMK if that is helpful.

If not, I can definitely repeat the EvalResponse for each test case.
created_at: 2025-04-17T14:39:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import * as _ from 'lodash';
path: genkit-tools/common/src/eval/parser.ts
user: login: pavelgj
body: Let's avoid adding lodash. Its use it increasingly discouraged.
Many of the things can be done with vanilla JS.
created_at: 2025-04-19T13:30:52Z
author_association: MEMBER
diff_hunk: @@ -113,4 +120,553 @@ describe('parser', () => {
       });
     });
   });
+
+  describe('extractMetricSummaries', () => {
+    const simpleEvalOutput: Record<string, EvalResponse> = {
+      '/evaluator/genkit/context_relevancy': [
+        {
+          testCaseId: 'case1',
+          evaluation: {
+            score: 7,
+          },
+        },
+        {
+          testCaseId: 'case2',
+          evaluation: {
+            score: 10,
+          },
+        },
+        {
+          testCaseId: 'case3',
+          evaluation: {
+            score: 5,
+          },
+        },
+      ],
+    };
+
+    describe('simpler scenarios', () => {
+      it('mean for simple numeric scores', () => {
+        const results = extractMetricSummaries(simpleEvalOutput);
+
+        expect(results).toHaveLength(1);
+
+        const result = results[0];
+        expect(result).toEqual({
+          evaluator: '/evaluator/genkit/context_relevancy',
+          errorCount: 0,
+          undefinedCount: 0,
+          statusDistribution: { undefined: 3 },
+          average: 22.0 / 3,
+        });
+      });
+
+      it('distribution for simple boolean scores', () => {
+        const booleanScores = reMapScores(simpleEvalOutput, (response, i) => ({
+          testCaseId: response.testCaseId,
+          evaluation: {
+            score: i % 2 === 0,
path: genkit-tools/common/tests/eval/parser_test.ts
user: login: shrutip90
body: Thank you, this is better, but I would prefer having a different helper like I suggested below: "Maybe instead of reMapScores you can have a helper that takes an array of Score objects and generates Record<string, EvalResponse> with testcaseIds. That might be concise enough and easier to read."

Please check if we can do this.
created_at: 2025-04-28T19:18:59Z
author_association: CONTRIBUTOR

PR comments:
user: login: pavelgj
created_at: 2025-04-19T13:25:56Z
body: nit: PR & commit messags title should be `feat(cli)` or `feat(cli/evals)`
author_association: MEMBER
user: login: ssbushi
created_at: 2025-04-22T14:15:21Z
body: > nit: PR & commit messags title should be feat(cli) or feat(cli/evals)

This is not specific to the CLI, but okay.

author_association: CONTRIBUTOR