Repository: tensorzero/tensorzero (language: rust)

Issue:
Require a "Function" selection before "Metric" selection in SFT UI and filter irrelevant metrics
1) The "Variant" dropdown is disabled until we select a function. The same behavior should apply to "Metric".

2) Remove any metrics that have zero relevant feedback from the dropdown (i.e. Curated Inferences = 0). This might require a more advanced ClickHouse query.


Discussion / hints:
@GabrielBianconi Can I take up this one?
Thanks @ChetanXpro !
Hey, to filter out all metrics with 0 Curated Inferences, this is what I am thinking, please correct me if I missed something.

Currently we fetch all functions and metrics from config and then send it to API to get feedback and other counts.

So to filter out metrics with no curated inferences, we might need to do something like this:

Just after user selects any function name, we will make an API call to ClickHouse to get all valid metrics for function which have any feedback, and then we will filter out config metrics and show them accordingly in the dropdown.
I think this is correct. You'll need to write a pretty involved ClickHouse query to actually get all the metrics with nonzero amounts of feedback per function (it'll involve selecting all the ids and episode_ids from the inference table and joining against each of `BooleanMetricFeedback`, `FloatMetricFeedback`, and `DemonstrationFeedback`'s `target_id` columns), then finding all the unique metric names + demonstration if it's nonempty in the the intersection. I can help with this if you'd like. You should be able to test this against the fixture data already in the database.
Thanks, I will give it a shot, will let you know if I run into any problems.
Sgtm!

On Tue, Jan 14, 2025 at 11:58 AM, Chetan < ***@***.*** > wrote:

> 
> 
> 
> 
> 
> 
> 
> Thanks, I will give it a shot, will let you know if I run into any
> problems.
> 
> 
> 
> —
> Reply to this email directly, view it on GitHub (
> https://github.com/tensorzero/tensorzero/issues/741#issuecomment-2590567294
> ) , or unsubscribe (
> https://github.com/notifications/unsubscribe-auth/ACGQ6BHFE5FQENFFZFAKIBL2KU62LAVCNFSM6AAAAABU5GQNEOVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDKOJQGU3DOMRZGQ
> ).
> You are receiving this because you commented. Message ID: <tensorzero/tensorzero/issues/741/2590567294
> @ github. com>
> 
> 
>

Fix patch:
diff --git a/ui/app/routes.ts b/ui/app/routes.ts
index b2139f463..6efd70394 100644
--- a/ui/app/routes.ts
+++ b/ui/app/routes.ts
@@ -10,6 +10,10 @@ export default [
     "api/curated_inferences/count",
     "routes/api/curated_inferences/count.route.ts",
   ),
+  route(
+    "api/function/:function_name/feedback_counts",
+    "routes/api/function/$function_name/feedback_counts.route.ts",
+  ),
   route(
     "observability/inferences",
     "routes/observability/inferences/route.tsx",
diff --git a/ui/app/routes/api/function/$function_name/feedback_counts.route.ts b/ui/app/routes/api/function/$function_name/feedback_counts.route.ts
new file mode 100644
index 000000000..d76280b92
--- /dev/null
+++ b/ui/app/routes/api/function/$function_name/feedback_counts.route.ts
@@ -0,0 +1,37 @@
+import type { LoaderFunctionArgs } from "react-router";
+import { getConfig } from "~/utils/config/index.server";
+import { getInferenceTableName } from "~/utils/clickhouse/common";
+import {
+  queryMetricsWithFeedback,
+  type MetricsWithFeedbackData,
+} from "~/utils/clickhouse/feedback";
+
+export async function loader({
+  params,
+}: LoaderFunctionArgs): Promise<Response> {
+  const functionName = params.function_name;
+
+  if (!functionName) {
+    return Response.json({ metrics: [] } as MetricsWithFeedbackData);
+  }
+
+  try {
+    const config = await getConfig();
+    const functionConfig = config.functions[functionName];
+    const inferenceTable = getInferenceTableName(functionConfig);
+
+    const result = await queryMetricsWithFeedback({
+      function_name: functionName,
+      inference_table: inferenceTable,
+      metrics: config.metrics,
+    });
+
+    return Response.json(result);
+  } catch (error) {
+    console.error("Error fetching metrics with feedback:", error);
+    throw new Response("Error fetching metrics with feedback", {
+      status: 500,
+      statusText: "Failed to fetch metrics with feedback",
+    });
+  }
+}
diff --git a/ui/app/routes/optimization/supervised-fine-tuning/MetricSelector.tsx b/ui/app/routes/optimization/supervised-fine-tuning/MetricSelector.tsx
index 03d4da33a..df6c04954 100644
--- a/ui/app/routes/optimization/supervised-fine-tuning/MetricSelector.tsx
+++ b/ui/app/routes/optimization/supervised-fine-tuning/MetricSelector.tsx
@@ -1,4 +1,4 @@
-import type { Control } from "react-hook-form";
+import { useWatch, useFormContext, type Control } from "react-hook-form";
 import type { SFTFormValues } from "./types";
 import { Config } from "~/utils/config";
 import { FormField, FormItem, FormLabel } from "~/components/ui/form";
@@ -12,6 +12,10 @@ import {
 import { Skeleton } from "~/components/ui/skeleton";
 import { Input } from "~/components/ui/input";
 import { MetricBadges } from "~/components/metric/MetricBadges";
+import { useEffect, useMemo } from "react";
+import { useFetcher } from "react-router";
+import type { MetricsWithFeedbackData } from "~/utils/clickhouse/feedback";
+import { Badge } from "~/components/ui/badge";
 
 type MetricSelectorProps = {
   control: Control<SFTFormValues>;
@@ -26,6 +30,37 @@ export function MetricSelector({
   curatedInferenceCount,
   config,
 }: MetricSelectorProps) {
+  const metricsFetcher = useFetcher<MetricsWithFeedbackData>();
+  const { getValues, setValue } = useFormContext<SFTFormValues>();
+
+  const functionValue = useWatch({
+    control,
+    name: "function",
+  });
+
+  useEffect(() => {
+    if (functionValue) {
+      metricsFetcher.load(
+        `/api/function/${encodeURIComponent(functionValue)}/feedback_counts`,
+      );
+    }
+  }, [functionValue]);
+
+  const validMetrics = useMemo(() => {
+    if (!metricsFetcher.data) return new Set<string>();
+    return new Set(metricsFetcher.data.metrics.map((m) => m.metric_name));
+  }, [metricsFetcher.data]);
+
+  const isLoading = metricsFetcher.state === "loading";
+
+  // Reset metric value if the selected function does not have the previously selected metric
+  useEffect(() => {
+    const metricValue = getValues("metric");
+    if (functionValue && metricValue && !validMetrics.has(metricValue)) {
+      setValue("metric", null);
+    }
+  }, [functionValue, validMetrics, getValues, setValue]);
+
   return (
     <FormField
       control={control}
@@ -41,9 +76,14 @@ export function MetricSelector({
                   field.onChange(metricValue);
                 }}
                 value={field.value ?? "none"}
+                disabled={!functionValue || isLoading}
               >
                 <SelectTrigger>
-                  <SelectValue placeholder="Select a metric" />
+                  <SelectValue
+                    placeholder={
+                      isLoading ? "Loading metrics..." : "Select a metric"
+                    }
+                  />
                 </SelectTrigger>
                 <SelectContent>
                   <SelectItem value="none">
@@ -51,16 +91,29 @@ export function MetricSelector({
                       <span>None</span>
                     </div>
                   </SelectItem>
-                  {Object.entries(config.metrics).map(([name, metric]) => (
-                    <SelectItem key={name} value={name}>
-                      <div className="flex w-full items-center justify-between">
-                        <span>{name}</span>
-                        <div className="ml-2">
-                          <MetricBadges metric={metric} />
-                        </div>
-                      </div>
-                    </SelectItem>
-                  ))}
+                  {Object.entries(config.metrics)
+                    .filter(([name]) => validMetrics.has(name))
+                    .map(([name, metric]) => {
+                      const metricFeedback = metricsFetcher.data?.metrics.find(
+                        (m) => m.metric_name === name,
+                      );
+
+                      return (
+                        <SelectItem key={name} value={name}>
+                          <div className="flex w-full items-center justify-between">
+                            <span>{name}</span>
+                            <div className="ml-2 flex items-center gap-2">
+                              {metricFeedback && (
+                                <Badge className="bg-gray-200 text-gray-800 dark:bg-gray-900 dark:text-gray-300">
+                                  Count: {metricFeedback.feedback_count}
+                                </Badge>
+                              )}
+                              <MetricBadges metric={metric} />
+                            </div>
+                          </div>
+                        </SelectItem>
+                      );
+                    })}
                 </SelectContent>
               </Select>
 
diff --git a/ui/app/utils/clickhouse/curation.ts b/ui/app/utils/clickhouse/curation.ts
index 93497719b..417aebab0 100644
--- a/ui/app/utils/clickhouse/curation.ts
+++ b/ui/app/utils/clickhouse/curation.ts
@@ -11,7 +11,9 @@ import {
 import { clickhouseClient } from "./common";
 import type { FunctionConfig } from "../config/function";
 
-function getInferenceJoinKey(metric_config: MetricConfig): InferenceJoinKey {
+export function getInferenceJoinKey(
+  metric_config: MetricConfig,
+): InferenceJoinKey {
   if ("level" in metric_config) {
     switch (metric_config.level) {
       case "inference":
diff --git a/ui/app/utils/clickhouse/feedback.ts b/ui/app/utils/clickhouse/feedback.ts
index 5021bb615..c15a4e286 100644
--- a/ui/app/utils/clickhouse/feedback.ts
+++ b/ui/app/utils/clickhouse/feedback.ts
@@ -1,7 +1,9 @@
 import { z } from "zod";
 import type { TableBounds } from "./common";
 import { data } from "react-router";
-import { clickhouseClient } from "./common";
+import { clickhouseClient, InferenceJoinKey } from "./common";
+import type { MetricConfig } from "~/utils/config/metric";
+import { getInferenceJoinKey } from "~/utils/clickhouse/curation";
 
 export const booleanMetricFeedbackRowSchema = z.object({
   type: z.literal("boolean"),
@@ -742,3 +744,175 @@ export async function countFeedbackByTargetId(
     booleanMetrics + commentFeedback + demonstrationFeedback + floatMetrics
   );
 }
+
+export const metricsWithFeedbackRowSchema = z
+  .object({
+    function_name: z.string(),
+    metric_name: z.string(),
+    metric_type: z.enum(["boolean", "float", "demonstration"]),
+    feedback_count: z.number(),
+  })
+  .strict();
+
+export const metricsWithFeedbackDataSchema = z
+  .object({
+    metrics: z.array(metricsWithFeedbackRowSchema),
+  })
+  .strict();
+
+export type MetricsWithFeedbackRow = z.infer<
+  typeof metricsWithFeedbackRowSchema
+>;
+export type MetricsWithFeedbackData = z.infer<
+  typeof metricsWithFeedbackDataSchema
+>;
+
+export async function queryMetricsWithFeedback(params: {
+  function_name: string;
+  inference_table: string;
+  metrics: Record<string, MetricConfig>;
+}): Promise<MetricsWithFeedbackData> {
+  const { function_name, inference_table, metrics } = params;
+
+  const inferenceMetrics = Object.entries(metrics)
+    .filter(([, metric]) => {
+      try {
+        return getInferenceJoinKey(metric) === InferenceJoinKey.ID;
+      } catch {
+        return false;
+      }
+    })
+    .map(([name]) => name);
+
+  const episodeMetrics = Object.entries(metrics)
+    .filter(([, metric]) => {
+      try {
+        return getInferenceJoinKey(metric) === InferenceJoinKey.EPISODE_ID;
+      } catch {
+        return false;
+      }
+    })
+    .map(([name]) => name);
+
+  const idInClause =
+    inferenceMetrics.length > 0
+      ? `IN ('${inferenceMetrics.join("','")}')`
+      : `= ''`;
+
+  const episodeIdInClause =
+    episodeMetrics.length > 0 ? `IN ('${episodeMetrics.join("','")}')` : `= ''`;
+
+  const query = `
+    WITH
+    boolean_inference_metrics AS (
+      SELECT
+        i.function_name,
+        bmf.metric_name,
+        'boolean' as metric_type,
+        COUNT(DISTINCT i.id) as feedback_count
+      FROM tensorzero.${inference_table} i
+      JOIN tensorzero.BooleanMetricFeedback bmf ON bmf.target_id = i.id
+      WHERE i.function_name = {function_name:String}
+        AND bmf.metric_name ${idInClause}
+      GROUP BY i.function_name, bmf.metric_name
+      HAVING feedback_count > 0
+    ),
+
+    boolean_episode_metrics AS (
+      SELECT
+        i.function_name,
+        bmf.metric_name,
+        'boolean' as metric_type,
+        COUNT(DISTINCT i.id) as feedback_count
+      FROM tensorzero.${inference_table} i
+      JOIN tensorzero.BooleanMetricFeedback bmf ON bmf.target_id = i.episode_id
+      WHERE i.function_name = {function_name:String}
+        AND bmf.metric_name ${episodeIdInClause}
+      GROUP BY i.function_name, bmf.metric_name
+      HAVING feedback_count > 0
+    ),
+
+    float_inference_metrics AS (
+      SELECT
+        i.function_name,
+        fmf.metric_name,
+        'float' as metric_type,
+        COUNT(DISTINCT i.id) as feedback_count
+      FROM tensorzero.${inference_table} i
+      JOIN tensorzero.FloatMetricFeedback fmf ON fmf.target_id = i.id
+      WHERE i.function_name = {function_name:String}
+        AND fmf.metric_name ${idInClause}
+      GROUP BY i.function_name, fmf.metric_name
+      HAVING feedback_count > 0
+    ),
+
+    float_episode_metrics AS (
+      SELECT
+        i.function_name,
+        fmf.metric_name,
+        'float' as metric_type,
+        COUNT(DISTINCT i.id) as feedback_count
+      FROM tensorzero.${inference_table} i
+      JOIN tensorzero.FloatMetricFeedback fmf ON fmf.target_id = i.episode_id
+      WHERE i.function_name = {function_name:String}
+        AND fmf.metric_name ${episodeIdInClause}
+      GROUP BY i.function_name, fmf.metric_name
+      HAVING feedback_count > 0
+    ),
+
+    demo_metrics AS (
+      SELECT
+        i.function_name,
+        'demonstration' as metric_name,
+        'demonstration' as metric_type,
+        COUNT(DISTINCT i.id) as feedback_count
+      FROM tensorzero.${inference_table} i
+      JOIN tensorzero.DemonstrationFeedback df ON df.inference_id = i.id
+      WHERE i.function_name = {function_name:String}
+      GROUP BY i.function_name
+      HAVING feedback_count > 0
+    )
+
+    SELECT
+      function_name,
+      metric_name,
+      metric_type,
+      toString(feedback_count) as feedback_count
+    FROM (
+      SELECT * FROM boolean_inference_metrics
+      UNION ALL
+      SELECT * FROM boolean_episode_metrics
+      UNION ALL
+      SELECT * FROM float_inference_metrics
+      UNION ALL
+      SELECT * FROM float_episode_metrics
+      UNION ALL
+      SELECT * FROM demo_metrics
+    )
+    ORDER BY metric_type, metric_name`;
+
+  try {
+    const resultSet = await clickhouseClient.query({
+      query,
+      format: "JSONEachRow",
+      query_params: { function_name },
+    });
+
+    const rawMetrics = (await resultSet.json()) as Array<{
+      function_name: string;
+      metric_name: string;
+      metric_type: "boolean" | "float" | "demonstration";
+      feedback_count: string;
+    }>;
+
+    const validMetrics = rawMetrics.map((metric) => ({
+      ...metric,
+      feedback_count: Number(metric.feedback_count),
+    }));
+
+    return { metrics: validMetrics };
+  } catch (error) {
+    console.error("Error fetching metrics with feedback:", error);
+    throw data("Error fetching metrics with feedback", { status: 500 });
+  }
+}


Test patch:
diff --git a/ui/app/utils/clickhouse/feedback.test.ts b/ui/app/utils/clickhouse/feedback.test.ts
index 7cc1adc9e..94ea6788b 100644
--- a/ui/app/utils/clickhouse/feedback.test.ts
+++ b/ui/app/utils/clickhouse/feedback.test.ts
@@ -1,9 +1,10 @@
-import { expect, test } from "vitest";
+import { describe, expect, test } from "vitest";
 import {
   countCommentFeedbackByTargetId,
   queryCommentFeedbackBoundsByTargetId,
   queryCommentFeedbackByTargetId,
   queryFeedbackByTargetId,
+  queryMetricsWithFeedback,
   type BooleanMetricFeedbackRow,
   type CommentFeedbackRow,
   type FeedbackRow,
@@ -216,3 +217,112 @@ test("queryFeedbackByTargetId", async () => {
       thirdBackwardPage.length,
   ).toBe(27);
 });
+
+describe("queryMetricsWithFeedback", () => {
+  test("returns correct feedback counts for different metric types", async () => {
+    // Test json function with multiple metric types
+    const jsonResults = await queryMetricsWithFeedback({
+      function_name: "dashboard_fixture_extract_entities",
+      inference_table: "JsonInference",
+      metrics: {
+        dashboard_fixture_exact_match: {
+          type: "boolean",
+          optimize: "max",
+          level: "inference",
+        },
+        dashboard_fixture_jaccard_similarity: {
+          type: "float",
+          optimize: "max",
+          level: "inference",
+        },
+      },
+    });
+
+    // Check boolean and demonstration counts for JSON function
+    expect(jsonResults.metrics).toContainEqual({
+      function_name: "dashboard_fixture_extract_entities",
+      metric_name: "dashboard_fixture_exact_match",
+      metric_type: "boolean",
+      feedback_count: 99,
+    });
+    expect(jsonResults.metrics).toContainEqual({
+      function_name: "dashboard_fixture_extract_entities",
+      metric_name: "demonstration",
+      metric_type: "demonstration",
+      feedback_count: 100,
+    });
+
+    // Test chat function with float metrics
+    const chatResults = await queryMetricsWithFeedback({
+      function_name: "dashboard_fixture_write_haiku",
+      inference_table: "ChatInference",
+      metrics: {
+        dashboard_fixture_haiku_rating: {
+          type: "float",
+          optimize: "max",
+          level: "inference",
+        },
+      },
+    });
+
+    expect(chatResults.metrics).toContainEqual({
+      function_name: "dashboard_fixture_write_haiku",
+      metric_name: "dashboard_fixture_haiku_rating",
+      metric_type: "float",
+      feedback_count: 491,
+    });
+  });
+
+  // Tests error handling for nonexistent functions
+  test("returns empty array for nonexistent function", async () => {
+    const results = await queryMetricsWithFeedback({
+      function_name: "nonexistent_function",
+      inference_table: "ChatInference",
+      metrics: {
+        dashboard_fixture_haiku_rating: {
+          type: "float",
+          optimize: "max",
+          level: "inference",
+        },
+      },
+    });
+
+    expect(results.metrics).toEqual([]);
+  });
+
+  // Tests handling of metrics at different levels (inference vs episode)
+  test("returns correct metrics for both inference and episode levels", async () => {
+    const results = await queryMetricsWithFeedback({
+      function_name: "dashboard_fixture_write_haiku",
+      inference_table: "ChatInference",
+      metrics: {
+        dashboard_fixture_haiku_rating: {
+          type: "float",
+          optimize: "max",
+          level: "inference",
+        },
+        dashboard_fixture_haiku_rating_episode: {
+          type: "float",
+          optimize: "max",
+          level: "episode",
+        },
+      },
+    });
+
+    // Check inference level metric
+    expect(results.metrics).toContainEqual({
+      function_name: "dashboard_fixture_write_haiku",
+      metric_name: "dashboard_fixture_haiku_rating",
+      metric_type: "float",
+      feedback_count: 491,
+    });
+
+    // Check episode level metric
+    expect(results.metrics).toContainEqual({
+      function_name: "dashboard_fixture_write_haiku",
+      metric_name: "dashboard_fixture_haiku_rating_episode",
+      metric_type: "float",
+      feedback_count: 85,
+    });
+  });
+});