add latency cost table

app.py

@@ -11,8 +11,11 @@ from src.about import (  # CITATION_BUTTON_LABEL,; CITATION_BUTTON_TEXT,; EVALUA
 from src.display.css_html_js import custom_css
 from src.display.utils import (  # EVAL_TYPES,; WeightType,; BENCHMARK_COLS,; EVAL_COLS,; NUMERIC_INTERVALS,; ModelType,; Precision,
     COLS,
+    COST_COLS,
+    COST_TYPES,
     TYPES,
     AutoEvalColumn,
+    CostEvalColumn,
     fields,
 )
 
@@ -20,10 +23,11 @@ from src.display.utils import (  # EVAL_TYPES,; WeightType,; BENCHMARK_COLS,; EV
 from src.envs import CRM_RESULTS_PATH
 from src.populate import get_leaderboard_df_crm
 
-original_df = get_leaderboard_df_crm(CRM_RESULTS_PATH, COLS)
+original_df, cost_df = get_leaderboard_df_crm(CRM_RESULTS_PATH, COLS, COST_COLS)
 
 # raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
+leaderboard_cost_df = cost_df.copy()
 # leaderboard_df = leaderboard_df.style.format({"accuracy_metric_average": "{0:.2f}"})
 
 
@@ -38,19 +42,12 @@ def update_table(
     use_case_area_query: list,
     use_case_query: list,
     use_case_type_query: list,
-    # type_query: list,
-    # precision_query: str,
-    # size_query: list,
-    # show_deleted: bool,
-    # query: str,
 ):
-    # filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-    # filtered_df = filter_queries(query, filtered_df)
     filtered_df = filter_llm_func(hidden_df, llm_query)
     filtered_df = filter_llm_provider_func(filtered_df, llm_provider_query)
     filtered_df = filter_accuracy_method_func(filtered_df, accuracy_method_query)
-    filtered_df = filter_accuracy_threshold_func(filtered_df, accuracy_threshold_query)
-
+    filtered_df["Accuracy Threshold"] = filter_accuracy_threshold_func(filtered_df, accuracy_threshold_query)
+    filtered_df = filtered_df[filtered_df["Accuracy Threshold"]]
     filtered_df["Use Case Area"] = filtered_df["Use Case Name"].apply(lambda x: x.split(": ")[0])
     filtered_df = filter_use_case_area_func(filtered_df, use_case_area_query)
     filtered_df = filter_use_case_func(filtered_df, use_case_query)
@@ -59,6 +56,32 @@ def update_table(
     return df
 
 
+def update_cost_table(
+    hidden_df: pd.DataFrame,
+    columns: list,
+    llm_query: list,
+    llm_provider_query: list,
+    use_case_type_query: list,
+):
+    filtered_df = filter_llm_func(hidden_df, llm_query)
+    filtered_df = filter_llm_provider_func(filtered_df, llm_provider_query)
+    filtered_df = filter_use_case_type_func(filtered_df, use_case_type_query)
+    df = select_columns_cost_table(filtered_df, columns)
+    return df.style.map(highlight_cost_band_low, props="background-color: #b3d5a4")
+
+
+# def highlight_cols(x):
+#     df = x.copy()
+#     df.loc[:, :] = "color: black"
+#     df.loc[, ["Accuracy"]] = "background-color: #b3d5a4"
+#     return df
+
+
+def highlight_cost_band_low(s, props=""):
+
+    return props if s == "Low" else None
+
+
 def init_leaderboard_df(
     leaderboard_df: pd.DataFrame,
     columns: list,
@@ -72,7 +95,6 @@ def init_leaderboard_df(
 ):
 
     # Applying the style function
-    # df = leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value]
     # return df.style.apply(highlight_cols, axis=None)
     return update_table(
         leaderboard_df,
@@ -87,13 +109,30 @@ def init_leaderboard_df(
     )
 
 
+def init_leaderboard_cost_df(
+    leaderboard_df: pd.DataFrame,
+    columns: list,
+    llm_query: list,
+    llm_provider_query: list,
+    use_case_type_query: list,
+):
+
+    return update_cost_table(
+        leaderboard_df,
+        columns,
+        llm_query,
+        llm_provider_query,
+        use_case_type_query,
+    )
+
+
 def filter_accuracy_method_func(df: pd.DataFrame, accuracy_method_query: str) -> pd.DataFrame:
     return df[df["Accuracy Method"] == accuracy_method_query]
 
 
 def filter_accuracy_threshold_func(df: pd.DataFrame, accuracy_threshold_query: str) -> pd.DataFrame:
     accuracy_cols = ["Instruction Following", "Conciseness", "Completeness", "Accuracy"]
-    return df[(df.loc[:, accuracy_cols] >= float(accuracy_threshold_query)).all(axis=1)]
+    return (df.loc[:, accuracy_cols] >= float(accuracy_threshold_query)).all(axis=1)
 
 
 def filter_use_case_area_func(df: pd.DataFrame, use_case_area_query: list) -> pd.DataFrame:
@@ -130,45 +169,12 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     return filtered_df
 
 
-# def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-#     final_df = []
-#     if query != "":
-#         queries = [q.strip() for q in query.split(";")]
-#         for _q in queries:
-#             _q = _q.strip()
-#             if _q != "":
-#                 temp_filtered_df = search_table(filtered_df, _q)
-#                 if len(temp_filtered_df) > 0:
-#                     final_df.append(temp_filtered_df)
-#         if len(final_df) > 0:
-#             filtered_df = pd.concat(final_df)
-#             filtered_df = filtered_df.drop_duplicates(
-#                 subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-#             )
-
-#     return filtered_df
-
-
-# def filter_models(
-#     df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-# ) -> pd.DataFrame:
-#     # Show all models
-#     filtered_df = df
-#     # if show_deleted:
-#     #     filtered_df = df
-#     # else:  # Show only still on the hub models
-#     #     filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True]
-
-#     type_emoji = [t[0] for t in type_query]
-#     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-#     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-#     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-#     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-#     mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-#     filtered_df = filtered_df.loc[mask]
-
-#     return filtered_df
+def select_columns_cost_table(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    always_here_cols = [
+        CostEvalColumn.model.name,
+    ]
+    filtered_df = df[always_here_cols + [c for c in COST_COLS if c in df.columns and c in columns]]
+    return filtered_df
 
 
 demo = gr.Blocks(css=custom_css)
@@ -259,14 +265,14 @@ with demo:
                 #         multiselect=True,
                 #         interactive=True,
                 #     )
-                with gr.Column():
-                    filter_metric_area = gr.CheckboxGroup(
-                        choices=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
-                        value=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
-                        label="Metric Area",
-                        info="",
-                        interactive=True,
-                    )
+                # with gr.Column():
+                #     filter_metric_area = gr.CheckboxGroup(
+                #         choices=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
+                #         value=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"],
+                #         label="Metric Area",
+                #         info="",
+                #         interactive=True,
+                #     )
                 with gr.Column():
                     filter_accuracy_method = gr.Radio(
                         choices=["Manual", "Auto"],
@@ -374,6 +380,87 @@ with demo:
                     leaderboard_table,
                     queue=True,
                 )
+        with gr.TabItem("🏅 Latency & Cost", elem_id="llm-benchmark-tab-table", id=1):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(CostEvalColumn) if not c.hidden and not c.never_hidden],
+                            value=[
+                                c.name
+                                for c in fields(CostEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+            with gr.Row():
+                with gr.Column():
+                    filter_llm = gr.CheckboxGroup(
+                        choices=list(cost_df["Model Name"].unique()),
+                        value=list(cost_df["Model Name"].unique()),
+                        label="Model Name",
+                        info="",
+                        interactive=True,
+                    )
+                with gr.Column():
+                    filter_llm_provider = gr.CheckboxGroup(
+                        choices=list(cost_df["LLM Provider"].unique()),
+                        value=list(cost_df["LLM Provider"].unique()),
+                        label="LLM Provider",
+                        info="",
+                        interactive=True,
+                    )
+                with gr.Column():
+                    filter_use_case_type = gr.CheckboxGroup(
+                        choices=["Long", "Short"],
+                        value=["Long", "Short"],
+                        label="Use Case Type",
+                        info="Output: 250 tokens, Long input: 3k tokens, Short input: 500 tokens",
+                        interactive=True,
+                    )
+
+            leaderboard_table = gr.components.Dataframe(
+                value=init_leaderboard_cost_df(
+                    leaderboard_cost_df,
+                    shown_columns.value,
+                    filter_llm.value,
+                    filter_llm_provider.value,
+                    filter_use_case_type.value,
+                ),
+                headers=[c.name for c in fields(CostEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=COST_TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=cost_df[COST_COLS],
+                headers=COST_COLS,
+                datatype=COST_TYPES,
+                visible=False,
+            )
+
+            for selector in [
+                shown_columns,
+                filter_llm,
+                filter_llm_provider,
+                filter_use_case_type,
+            ]:
+                selector.change(
+                    update_cost_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        filter_llm,
+                        filter_llm_provider,
+                        filter_use_case_type,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

crm-results/hf_leaderboard_latency_cost.csv

@@ -1,37 +1,37 @@
-Model Name,Use Case Type (Long vs Short),Platform,Mean Latency (sec) per Request,Mean Output Tokens,Mean Cost per 1K Requests,Cost Band
-AI21 Jamba-Instruct,Long,AI21,4.0,232.9,1.6,High
-AI21 Jamba-Instruct,Short,AI21,4.0,243.9,0.5,High
-Claude 3 Haiku,Long,Bedrock,2.8,236.9,1.0,High
-Claude 3 Haiku,Short,Bedrock,2.2,245.4,0.4,High
-Claude 3 Opus,Long,Bedrock,12.2,242.7,61.1,High
-Claude 3 Opus,Short,Bedrock,8.4,243.2,25.4,High
-Cohere Command R+,Long,Bedrock,7.7,245.7,11.7,High
-Cohere Command R+,Short,Bedrock,7.1,249.9,5.1,High
-Cohere Command Text,Long,Bedrock,12.9,238.7,4.3,High
-Cohere Command Text,Short,Bedrock,9.6,245.6,1.1,High
-Gemini Pro 1.5,Long,Google,5.5,245.7,11.0,High
-Gemini Pro 1.5,Short,Google,5.4,247.5,3.3,High
-Gemini Pro 1,Long,Google,6.0,228.9,1.7,High
-Gemini Pro 1,Short,Google,4.4,247.4,0.6,High
-GPT 3.5 Turbo,Long,OpenAI,4.5,249.9,1.6,High
-GPT 3.5 Turbo,Short,OpenAI,4.2,238.3,0.6,High
-GPT 4 Turbo,Long,OpenAI,12.3,247.6,32.0,High
-GPT 4 Turbo,Short,OpenAI,12.3,250.0,11.7,High
-GPT4-o,Long,OpenAI,5.1,248.4,15.9,High
-GPT4-o,Short,OpenAI,5.0,250.0,5.8,High
-Mistral 7B,Long,Self-host (g5.48xlarge),8.83,242.0,16.5,High
-Mistral 7B,Short,Self-host (g5.48xlarge),8.31,247.0,15.5,High
-LLaMA 3 8B,Long,Self-host (g5.48xlarge),3.76,251.5,7.0,High
-LLaMA 3 8B,Short,Self-host (g5.48xlarge),3.23,243.6,6.0,High
-LLaMA 3 70B,Long,Self-host (p4d.24xlarge),20.1,243.9,67.7,High
-LLaMA 3 70B,Short,Self-host (p4d.24xlarge),29.4,251.2,99.0,High
-Mixtral 8x7B,Long,Self-host (p4d.24xlarge),2.44,248.5,8.22,High
-Mixtral 8x7B,Short,Self-host (p4d.24xlarge),2.41,250.0,8.11,High
-SF-TextBase 7B,Long,Self-host (g5.48xlarge),8.99,248.5,16.80,High
-SF-TextBase 7B,Short,Self-host (g5.48xlarge),8.29,248.7,15.50,High
-SF-TextBase 70B,Long,Self-host (p4de.24xlarge),6.52,253.7,28.17,High
-SF-TextBase 70B,Short,Self-host (p4de.24xlarge),6.24,249.7,26.96,High
-SF-TextSum,Long,Self-host (g5.48xlarge),8.85,244.0,16.55,High
-SF-TextSum,Short,Self-host (g5.48xlarge),8.34,250.4,15.60,High
-XGen 2,Long,Self-host (p4de.24xlarge),3.71,250.0,16.03,High
-XGen 2,Short,Self-host (p4de.24xlarge),2.64,250.0,11.40,High
+Model Name,Use Case Type,Version,Platform,Mean Latency (sec) per Request,Mean Output Tokens,Mean Cost per 1K Requests,Cost Band,,Model id,Cost per 1m input tokens,Cost per 1m output tokens,,,,Percentile,From,To,,min,Max
+AI21 Jamba-Instruct,Long,,AI21,4.0,232.9,1.6,Medium,,GPT 3.5 Turbo,0.5,1.5,,,0%,0.43,0.43,1.61,,0.43,61.11
+AI21 Jamba-Instruct,Short,,AI21,4.0,243.9,0.5,Low,,GPT 4 Turbo,10,30,,,33%,1.61,1.61,9.28,,,
+Claude 3 Haiku,Long,,Bedrock,2.8,236.9,1.0,Low,,GPT4-o,5,15,,,67%,9.28,9.28,61.11,,,
+Claude 3 Haiku,Short,,Bedrock,2.2,245.4,0.4,Low,,Claude 3 Haiku,0.25,1.25,,,100%,61.11,,,,,
+Claude 3 Opus,Long,,Bedrock,12.2,242.7,61.1,High,,Claude 3 Opus,15,75,,,,,,,,,
+Claude 3 Opus,Short,,Bedrock,8.4,243.2,25.4,High,,AI21 Jamba-Instruct,0.5,0.7,,,,,,,,,
+Cohere Command R+,Long,,Bedrock,7.7,245.7,11.7,High,,Cohere Command Text,1.5,2,,,,,,,,,
+Cohere Command R+,Short,,Bedrock,7.1,249.9,5.1,Medium,,Cohere Command R+,3,15,,,,,,,,,
+Cohere Command Text,Long,,Bedrock,12.9,238.7,4.3,Medium,,Gemini Pro 1,0.5,1.5,,,,,,,,,
+Cohere Command Text,Short,,Bedrock,9.6,245.6,1.1,Low,,Gemini Pro 1.5,3.5,7,,,,,,,,,
+Gemini Pro 1.5,Long,,Google,5.5,245.7,11.0,High,,,,,,,,,,,,,
+Gemini Pro 1.5,Short,,Google,5.4,247.5,3.3,Medium,,,,,,,,,,,,,
+Gemini Pro 1,Long,,Google,6.0,228.9,1.7,Medium,,,,,,,,,,,,,
+Gemini Pro 1,Short,,Google,4.4,247.4,0.6,Low,,,,,,,,,,,,,
+GPT 3.5 Turbo,Long,,OpenAI,4.5,249.9,1.6,Low,,,,,,,,,,,,,
+GPT 3.5 Turbo,Short,,OpenAI,4.2,238.3,0.6,Low,,,,,,,,,,,,,
+GPT 4 Turbo,Long,,OpenAI,12.3,247.6,32.0,High,,,,,,,,,,,,,
+GPT 4 Turbo,Short,,OpenAI,12.3,250.0,11.7,High,,,,,,,,,,,,,
+GPT4-o,Long,,OpenAI,5.1,248.4,15.9,High,,,,,,,,,,,,,
+GPT4-o,Short,,OpenAI,5.0,250.0,5.8,Medium,,,,,,,,,,,,,
+Mistral 7B,Long,Mistral-7B-Instruct-v0.2,Self-host (g5.48xlarge),8.83,242.0,16.5,High,,,,,,,,,,,,,
+Mistral 7B,Short,Mistral-7B-Instruct-v0.2,Self-host (g5.48xlarge),8.31,247.0,15.5,High,,,,,,,,,,,,,
+LLaMA 3 8B,Long,Meta-Llama-3-8B-Instruct,Self-host (g5.48xlarge),3.76,251.5,7.0,Medium,,,,,,,,,,,,,
+LLaMA 3 8B,Short,Meta-Llama-3-8B-Instruct,Self-host (g5.48xlarge),3.23,243.6,6.0,Medium,,,,,,,,,,,,,
+LLaMA 3 70B,Long,llama-3-70b-instruct,Self-host (p4d.24xlarge),20.1,243.9,67.7,High,,,,,,,,,,,,,
+LLaMA 3 70B,Short,llama-3-70b-instruct,Self-host (p4d.24xlarge),29.4,251.2,99.0,High,,,,,,,,,,,,,
+Mixtral 8x7B,Long,mixtral-8x7b-instruct,Self-host (p4d.24xlarge),2.44,248.5,8.22,Medium,,,,,,,,,,,,,
+Mixtral 8x7B,Short,mixtral-8x7b-instruct,Self-host (p4d.24xlarge),2.41,250.0,8.11,Medium,,,,,,,,,,,,,
+SF-TextBase 7B,Long,CRM-TextBase-7b-22k-g5 (endpoint),Self-host (g5.48xlarge),8.99,248.5,16.80,High,,,,,,,,,,,,,
+SF-TextBase 7B,Short,CRM-TextBase-7b-22k-g5 (endpoint),Self-host (g5.48xlarge),8.29,248.7,15.50,High,,,,,,,,,,,,,
+SF-TextBase 70B,Long,TextBase-70B-8K,Self-host (p4de.24xlarge),6.52,253.7,28.17,High,,,,,,,,,,,,,
+SF-TextBase 70B,Short,TextBase-70B-8K,Self-host (p4de.24xlarge),6.24,249.7,26.96,High,,,,,,,,,,,,,
+SF-TextSum,Long,CRM-TSUM-7b-22k-g5 (endpoint),Self-host (g5.48xlarge),8.85,244.0,16.55,High,,,,,,,,,,,,,
+SF-TextSum,Short,CRM-TSUM-7b-22k-g5 (endpoint),Self-host (g5.48xlarge),8.34,250.4,15.60,High,,,,,,,,,,,,,
+XGen 22B,Long,EinsteinXgen2E4DSStreaming (endpoint),Self-host (p4de.24xlarge),3.71,250.0,16.03,High,not able to get response for large token requests (5K-token input),,,,,,,,,,,,
+XGen 22B,Short,EinsteinXgen2E4DSStreaming (endpoint),Self-host (p4de.24xlarge),2.64,250.0,11.40,High,,,,,,,,,,,,,

src/display/utils.py

@@ -51,9 +51,26 @@ auto_eval_column_dict.append(
 auto_eval_column_dict.append(
     ["accuracy_metric_factuality", ColumnContent, ColumnContent("Factuality", "markdown", True)]
 )
-# Speed (Latency) metrics
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
 
-# Cost metrics
+# Speed (Latency) & Cost metrics
+cost_eval_column_dict = []
+# Init
+cost_eval_column_dict.append(
+    ["model", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)]
+)
+cost_eval_column_dict.append(["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True)])
+cost_eval_column_dict.append(["use_case_type", ColumnContent, ColumnContent("Use Case Type", "markdown", True)])
+cost_eval_column_dict.append(
+    ["latency", ColumnContent, ColumnContent("Mean Latency (sec) per Request", "markdown", True)]
+)
+cost_eval_column_dict.append(
+    ["mean_output_tokens", ColumnContent, ColumnContent("Mean Output Tokens", "markdown", True)]
+)
+cost_eval_column_dict.append(["cost_band", ColumnContent, ColumnContent("Cost Band", "markdown", True)])
+CostEvalColumn = make_dataclass("CostEvalColumn", cost_eval_column_dict, frozen=True)
 
 # Trust & Safety metrics
 
@@ -73,9 +90,6 @@ auto_eval_column_dict.append(
 # auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 # auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
 
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
@@ -156,6 +170,9 @@ TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 
+COST_COLS = [c.name for c in fields(CostEvalColumn) if not c.hidden]
+COST_TYPES = [c.type for c in fields(CostEvalColumn) if not c.hidden]
+
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 

src/populate.py

@@ -1,17 +1,19 @@
-import json
 import os
 
 import pandas as pd
 
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+# from src.display.formatting import has_no_nan_values, make_clickable_model
+# from src.display.utils import AutoEvalColumn, EvalQueueColumn
+# from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df_crm(crm_results_path: str, cols: list) -> pd.DataFrame:
+def get_leaderboard_df_crm(
+    crm_results_path: str, accuracy_cols: list, cost_cols: list
+) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Creates a dataframe from all the individual experiment results"""
-    leaderboard_accuracy_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_accuracy.csv"))
     sf_finetuned_models = ["SF-TextBase 70B", "SF-TextBase 7B", "SF-TextSum"]
+
+    leaderboard_accuracy_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_accuracy.csv"))
     leaderboard_accuracy_df = leaderboard_accuracy_df[~leaderboard_accuracy_df["Model Name"].isin(sf_finetuned_models)]
     # leaderboard_accuracy_df = leaderboard_accuracy_df.sort_values(
     #     by=[AutoEvalColumn.accuracy_metric_average.name], ascending=False
@@ -19,55 +21,62 @@ def get_leaderboard_df_crm(crm_results_path: str, cols: list) -> pd.DataFrame:
     # print(leaderboard_accuracy_df)
     # print(leaderboard_accuracy_df.columns)
     # print(leaderboard_accuracy_df["Model Name"].nunique())
-    leaderboard_accuracy_df = leaderboard_accuracy_df[cols].round(decimals=2)
-    return leaderboard_accuracy_df
+    leaderboard_accuracy_df = leaderboard_accuracy_df[accuracy_cols].round(decimals=2)
 
+    ref_df = leaderboard_accuracy_df[["Model Name", "LLM Provider"]].drop_duplicates()
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return raw_data, df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+    leaderboard_cost_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_latency_cost.csv"))
+    leaderboard_cost_df = leaderboard_cost_df[~leaderboard_cost_df["Model Name"].isin(sf_finetuned_models)]
+    leaderboard_cost_df = leaderboard_cost_df.join(ref_df.set_index("Model Name"), on="Model Name")
+    leaderboard_cost_df["LLM Provider"] = leaderboard_cost_df["LLM Provider"].fillna("Google")
+    leaderboard_cost_df = leaderboard_cost_df[cost_cols].round(decimals=2)
+    return leaderboard_accuracy_df, leaderboard_cost_df
+
+
+# def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+#     """Creates a dataframe from all the individual experiment results"""
+#     raw_data = get_raw_eval_results(results_path, requests_path)
+#     all_data_json = [v.to_dict() for v in raw_data]
+
+#     df = pd.DataFrame.from_records(all_data_json)
+#     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+#     df = df[cols].round(decimals=2)
+
+#     # filter out if any of the benchmarks have not been produced
+#     df = df[has_no_nan_values(df, benchmark_cols)]
+#     return raw_data, df
+
+# def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+#     """Creates the different dataframes for the evaluation queues requestes"""
+#     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+#     all_evals = []
+
+#     for entry in entries:
+#         if ".json" in entry:
+#             file_path = os.path.join(save_path, entry)
+#             with open(file_path) as fp:
+#                 data = json.load(fp)
+
+#             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+#             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+#             all_evals.append(data)
+#         elif ".md" not in entry:
+#             # this is a folder
+#             sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+#             for sub_entry in sub_entries:
+#                 file_path = os.path.join(save_path, entry, sub_entry)
+#                 with open(file_path) as fp:
+#                     data = json.load(fp)
+
+#                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+#                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+#                 all_evals.append(data)
+
+#     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+#     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+#     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+#     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+#     df_running = pd.DataFrame.from_records(running_list, columns=cols)
+#     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+#     return df_finished[cols], df_running[cols], df_pending[cols]