New layout

app.py

@@ -3,19 +3,21 @@ from os import listdir
 from os.path import isfile, join
 import numpy as np  # np mean, np random
 import pandas as pd  # read csv, df manipulation
+from collections import Counter
 import plotly.express as px  # interactive charts
 from plotly import graph_objs as go
 import streamlit as st  # 🎈 data web app development
 import plotly.figure_factory as ff
 import numpy as np
 from collections import Counter
-
+from streamlit_echarts import st_echarts
+import streamlit_toggle as tog
 
 print("Make sure to activate your VPN before running this script")
 
 st.set_page_config(
-    page_title="GroqFlow Progress Tracker",
-    page_icon="🚀",
+    page_title="ML Agility tracker",
+    page_icon="⚡",
     layout="wide",
 )
 
@@ -26,28 +28,67 @@ if "INFO_CLOSED" not in state:
     state.INFO_CLOSED = False
 
 # dashboard title
-st.title("GroqFlow Progress Tracker 🚀")
+st.title("ML Agility Tracker ⚡")
 
 # Custom chart colors (https://plotly.com/python/discrete-color/)
-colorway = ["#3366cc", "#FF7F0E"]
+colorway = [
+    "#5470c6",
+    "#FF7F0E",
+    "#94cc74",
+    "#92cb75",
+    "#fac858",
+    "#ee6666",
+    "#73c0de",
+    "#3ba272",
+]
+# colorway = ["#3366cc", "#FF7F0E"]
+
+st.markdown(
+    "Machine Learning Agility (MLAgility) measures vendor progress towards providing this turnkey solution to their customers. For more details, please visit [mlagility.org](mlagility.org).",
+    unsafe_allow_html=True,
+)
+
+
+def add_filter(
+    data_frame_list, name, label, options=None, num_cols=1, last_is_others=True
+):
 
+    # Get list of all options and return if no options are available
+    all_options = set(data_frame_list[-1][label])
+    if "-" in all_options:
+        all_options.remove("-")
+    if len(all_options) == 0:
+        return data_frame_list
 
-def add_filter(data_frame_list, name, label, options, num_cols=1):
     st.markdown(f"#### {name}")
 
+    # Create list of options if selectable options are not provided
+    if options is None:
+        options_dict = Counter(data_frame_list[-1][label])
+        sorted_options = sorted(options_dict, key=options_dict.get, reverse=True)
+        if "-" in sorted_options:
+            sorted_options.remove("-")
+        if len(sorted_options) > 8:
+            options = list(sorted_options[:7]) + ["others"]
+            last_is_others = True
+        else:
+            options = list(sorted_options)
+            last_is_others = False
+
     cols = st.columns(num_cols)
     instantiated_checkbox = []
     for idx in range(len(options)):
         with cols[idx % num_cols]:
-            instantiated_checkbox.append(st.checkbox(options[idx], False))
+            instantiated_checkbox.append(
+                st.checkbox(options[idx], False, key=f"{label}_{options[idx]}")
+            )
 
-    all_options = set(data_frame_list[-1][label])
     selected_options = [
         options[idx] for idx, checked in enumerate(instantiated_checkbox) if checked
     ]
 
     # The last checkbox will always correspond to "other"
-    if instantiated_checkbox[-1]:
+    if instantiated_checkbox[-1] and last_is_others:
         selected_options = selected_options[:-1]
         other_options = [x for x in all_options if x not in options]
         selected_options = set(selected_options + other_options)
@@ -63,21 +104,34 @@ def add_filter(data_frame_list, name, label, options, num_cols=1):
     return data_frame_list
 
 
-with st.sidebar:
+def parameter_filter(data_frame_list):
 
-    st.markdown("# Filters")
+    st.markdown(f"#### Parameters")
 
-    test_type = st.radio(
-        "Test Type",
-        ("Daily Tests (100 models)", "Monthly Tests (500+ models)"),
+    start_params, end_params = st.select_slider(
+        "Select a range parameters (in millions)",
+        options=[str(x) for x in np.arange(0, 1001, 10, dtype=int)],
+        value=("0", "1000"),
     )
-    if test_type == "Daily Tests (100 models)":
-        selected_test_type = "daily"
-        report_folder = "reports/daily"
-    else:
-        selected_test_type = "monthly"
-        report_folder = "reports/monthly"
 
+    for idx in range(len(data_frame_list)):
+        data_frame_list[idx] = data_frame_list[idx][
+            [
+                int(model_entry) >= int(start_params) * 1000000
+                and int(model_entry) <= int(end_params) * 1000000
+                for model_entry in data_frame_list[idx]["params"]
+            ]
+        ]
+
+    return data_frame_list
+
+
+with st.sidebar:
+
+    st.markdown("# Filters")
+
+    selected_test_type = "mlagility"
+    report_folder = "reports/mlagility"
     # Get ML Agility reports
     reports = sorted(
         [f for f in listdir(report_folder) if isfile(join(report_folder, f))]
@@ -89,29 +143,29 @@ with st.sidebar:
     mla_report = pd.read_csv(f"{report_folder}/{selected_report}")
     prev_mla_report = pd.read_csv(f"{report_folder}/{prev_report}")
 
-    # Add chips filter
-    num_chips_options = ["1", "2", "4", "8", "16", "32+"]
-    mla_report = mla_report.astype({"chips_used": str})
-    prev_mla_report = prev_mla_report.astype({"chips_used": str})
-    mla_report, prev_mla_report = add_filter(
-        [mla_report, prev_mla_report],
-        "Number of GroqChips™",
-        label="chips_used",
-        options=num_chips_options,
-        num_cols=3,
-    )
+    # Convert int parameters to int/float
+    for p in ["chips_used", "cycles", "params"]:
+        mla_report[p] = mla_report[p].replace("-", 0).astype("int64")
+        prev_mla_report[p] = prev_mla_report[p].replace("-", 0).astype("int64")
+
+    # Parameter filter
+    mla_report, prev_mla_report = parameter_filter([mla_report, prev_mla_report])
 
     # Add author filter
-    authors = [
-        "google",
-        "apple",
-        "facebook",
-        "openai",
-        "microsoft",
-        "huggingface",
-        "CompVis",
-        "others",
-    ]
+    authors = (
+        [
+            "google",
+            "apple",
+            "facebook",
+            "openai",
+            "microsoft",
+            "huggingface",
+            "CompVis",
+            "others",
+        ]
+        if selected_test_type == "monthly"
+        else None
+    )
     mla_report, prev_mla_report = add_filter(
         [mla_report, prev_mla_report],
         "Authors",
@@ -134,6 +188,7 @@ with st.sidebar:
         "Summarization",
         "other",
     ]
+    tasks = None
     mla_report, prev_mla_report = add_filter(
         [mla_report, prev_mla_report], "Tasks", label="task", options=tasks
     )
@@ -168,221 +223,340 @@ placeholder = st.empty()
 with placeholder.container():
 
     st.markdown("## Summary Results")
-    # create three columns
-    kpi = st.columns(7)
-    model_details = st.columns(7)
-
-    # fill in those three columns with respective metrics or KPIs
-    kpi[0].metric(
-        label="All models",
-        value=len(mla_report),
-        delta=len(mla_report) - len(prev_mla_report),
-    )
-    if selected_test_type == "daily":
-        with model_details[0]:
-            detailed_progress_list(mla_report, prev_mla_report)
-
-    kpi[1].metric(
-        label="Convert to ONNX",
-        value=np.sum(mla_report["base_onnx"]),
-        delta=int(
-            np.sum(mla_report["base_onnx"]) - np.sum(prev_mla_report["base_onnx"])
-        ),
-    )
-    if selected_test_type == "daily":
-        with model_details[1]:
-            detailed_progress_list(mla_report, prev_mla_report, "base_onnx")
-
-    kpi[2].metric(
-        label="Optimize ONNX file",
-        value=np.sum(mla_report["optimized_onnx"]),
-        delta=int(
-            np.sum(mla_report["optimized_onnx"])
-            - np.sum(prev_mla_report["optimized_onnx"])
-        ),
-    )
-    if selected_test_type == "daily":
-        with model_details[2]:
-            detailed_progress_list(mla_report, prev_mla_report, "optimized_onnx")
-
-    kpi[3].metric(
-        label="All ops supported",
-        value=np.sum(mla_report["all_ops_supported"]),
-        delta=int(
-            np.sum(mla_report["all_ops_supported"])
-            - np.sum(prev_mla_report["all_ops_supported"])
-        ),
-    )
-    if selected_test_type == "daily":
-        with model_details[3]:
-            detailed_progress_list(mla_report, prev_mla_report, "all_ops_supported")
-
-    kpi[4].metric(
-        label="Converts to FP16",
-        value=np.sum(mla_report["fp16_onnx"]),
-        delta=int(
-            np.sum(mla_report["fp16_onnx"]) - np.sum(prev_mla_report["fp16_onnx"])
-        ),
-    )
-    if selected_test_type == "daily":
-        with model_details[4]:
-            detailed_progress_list(mla_report, prev_mla_report, "fp16_onnx")
-
-    kpi[5].metric(
-        label="Compiles",
-        value=np.sum(mla_report["compiles"]),
-        delta=int(np.sum(mla_report["compiles"]) - np.sum(prev_mla_report["compiles"])),
-    )
-    if selected_test_type == "daily":
-        with model_details[5]:
-            detailed_progress_list(mla_report, prev_mla_report, "compiles")
-
-    kpi[6].metric(
-        label="Assembles",
-        value=np.sum(mla_report["assembles"]),
-        delta=int(
-            np.sum(mla_report["assembles"]) - np.sum(prev_mla_report["assembles"])
-        ),
-    )
-    if selected_test_type == "daily":
-        with model_details[6]:
-            detailed_progress_list(mla_report, prev_mla_report, "assembles")
+
+    all_models = len(mla_report)
+    base_onnx = np.sum(mla_report["base_onnx"])
+    optimized_onnx = np.sum(mla_report["optimized_onnx"])
+    all_ops_supported = np.sum(mla_report["all_ops_supported"])
+    fp16_onnx = np.sum(mla_report["fp16_onnx"])
+    compiles = np.sum(mla_report["compiles"])
+    assembles = np.sum(mla_report["assembles"])
+
+    # Pie chart for showing origin of models
+    # based on https://echarts.apache.org/examples/en/editor.html?c=pie-simple
+
+    all_authors = list(mla_report.loc[:, "author"])
+    try:
+        all_sources = list(mla_report.loc[:, "model_type"])
+    except KeyError:
+        all_sources = []
+    all_sources = []
+    author_count = {i: all_authors.count(i) for i in all_authors}
+    sources_count = {i: all_sources.count(i) for i in all_sources}
 
     cols = st.columns(2)
     with cols[0]:
-
-        compiler_errors = mla_report[mla_report["compiler_error"] != "-"][
-            "compiler_error"
-        ]
-        compiler_errors = Counter(compiler_errors)
-        st.markdown("""#### Top compiler issues""")
-        if len(compiler_errors) > 0:
-            compiler_errors = pd.DataFrame.from_dict(
-                compiler_errors, orient="index"
-            ).reset_index()
-            compiler_errors = compiler_errors.set_axis(
-                ["error", "count"], axis=1, inplace=False
-            )
-
-            fig = px.bar(
-                compiler_errors, x="count", y="error", orientation="h", height=400
-            )
-            st.plotly_chart(fig, use_container_width=True)
-        else:
-            st.markdown("""No compiler errors found :tada:""")
+        st.markdown("""#### Workload origin""")
+
+        options = {
+            "darkMode": "true",
+            "textStyle": {"fontSize": 16},
+            "tooltip": {"trigger": "item"},
+            "series": [
+                {
+                    "name": "Access From",
+                    "type": "pie",
+                    "radius": [0, "30%"],
+                    "label": {"position": "inner", "fontSize": 14},
+                    "labelLine": {"show": "false"},
+                    "data": [
+                        {"value": sources_count[k], "name": k}
+                        for k in sources_count.keys()
+                    ],
+                },
+                {
+                    "name": "Name of corpus:",
+                    "type": "pie",
+                    "radius": ["70%", "70%"],
+                    "data": [
+                        {"value": author_count[k], "name": k}
+                        for k in author_count.keys()
+                    ],
+                    "label": {
+                        "formatter": "{b}\n{d}%",
+                    },
+                },
+                {
+                    "name": "Name of corpus:",
+                    "type": "pie",
+                    "radius": ["50%", "70%"],
+                    "data": [
+                        {"value": author_count[k], "name": k}
+                        for k in author_count.keys()
+                    ],
+                    "emphasis": {
+                        "itemStyle": {
+                            "shadowBlur": 10,
+                            "shadowOffsetX": 0,
+                            "shadowColor": "rgba(0, 0, 0, 0.5)",
+                        }
+                    },
+                    "label": {
+                        "position": "inner",
+                        "formatter": "{c}",
+                        "color": "black",
+                        "textBorderWidth": 0,
+                    },
+                },
+                {
+                    # Show total number of models inside
+                    "name": "Total number of models:",
+                    "type": "pie",
+                    "radius": ["0%", "0%"],
+                    "data": [{"value": all_models, "name": "Total"}],
+                    "silent": "true",
+                    "label": {
+                        "position": "inner",
+                        "formatter": "{c}",
+                        "color": "white",
+                        "fontSize": 30,
+                        "textBorderWidth": 0,
+                    },
+                },
+            ],
+        }
+        st_echarts(
+            options=options,
+            height="400px",
+        )
 
     with cols[1]:
         # Add parameters histogram
         all_models = [float(x) / 1000000 for x in mla_report["params"] if x != "-"]
 
-        assembled_models = mla_report[mla_report["assembles"] == True]
-        assembled_models = [
-            float(x) / 1000000 for x in assembled_models["params"] if x != "-"
-        ]
         hist_data = []
         group_labels = []
         if all_models != []:
             hist_data.append(all_models)
-            group_labels.append("Models we tried compiling")
-
-        if assembled_models != []:
-            hist_data.append(assembled_models)
-            group_labels.append("Assembled models")
-
-        st.markdown("""#### Assembled models vs. Parameters (in millions)""")
+            group_labels.append("All models")
 
-        if len(assembled_models) > 1:
+        st.markdown("""#### Parameter Size Distribution""")
 
+        if hist_data != []:
             fig = ff.create_distplot(
                 hist_data,
                 group_labels,
-                bin_size=[25, 25],
+                bin_size=25,
                 histnorm="",
+                colors=colorway,
+                curve_type="normal",
             )
-            # fig.layout.update(title="Assembled models vs. Parameters (in millions)")
             fig.layout.update(xaxis_title="Parameters in millions")
             fig.layout.update(yaxis_title="count")
             fig.update_xaxes(range=[1, 1000])
             st.plotly_chart(fig, use_container_width=True)
+
         else:
-            st.markdown("""Need at least one assembled model to show this graph 😅""")
+            st.markdown(
+                """At least one model needs to reach the compiler to show this graph 😅"""
+            )
 
     if "tsp_gpu_compute_ratio" in mla_report and "tsp_gpu_e2e_ratio" in mla_report:
         cols = st.columns(2)
         with cols[0]:
             # GPU Acceleration plot
-            st.markdown("""#### Speedup of GroqChip™ compared to A100 GPUs""")
+            st.markdown("""#### Benchmark results (latency)""")
 
             # Prepare data
             df = mla_report[
-                ["model_name", "tsp_gpu_compute_ratio", "tsp_gpu_e2e_ratio"]
+                [
+                    "model_name",
+                    "tsp_estimated_e2e_latency",
+                    "gpu_e2e_latency",
+                ]
             ]
             df = df.sort_values(by=["model_name"])
-            df = df[(df.tsp_gpu_compute_ratio != "-")]
-            df = df[(df.tsp_gpu_e2e_ratio != "-")]
-            df["tsp_gpu_compute_ratio"] = df["tsp_gpu_compute_ratio"].astype(float)
-            df["tsp_gpu_e2e_ratio"] = df["tsp_gpu_e2e_ratio"].astype(float)
-
-            data = [
-                go.Bar(
-                    x=df["model_name"],
-                    y=df["tsp_gpu_compute_ratio"],
-                    name="Compute only",
-                ),
-                go.Bar(
-                    x=df["model_name"],
-                    y=df["tsp_gpu_e2e_ratio"],
-                    name="Compute + estimated I/O",
-                ),
-            ]
-
-            layout = go.Layout(
-                barmode="overlay",
-                yaxis_title="Speedup compared to A100 GPU",
-                colorway=colorway,
+            df = df[(df.tsp_estimated_e2e_latency != "-")]
+            df = df[(df.gpu_e2e_latency != "-")]
+            df["tsp_estimated_e2e_latency"] = df["tsp_estimated_e2e_latency"].astype(
+                float
             )
+            df["gpu_e2e_latency"] = df["gpu_e2e_latency"].astype(float)
+
+            if len(df) == 0 and assembles > 0:
+                st.markdown(
+                    (
+                        "We do not have GPU numbers for the model(s) mapped to the GroqChip."
+                        " This is potentially due to lack of out-of-the-box TensorRT support."
+                    )
+                )
+            elif assembles == 0:
+                st.markdown(
+                    "Nothing to show here since no models have been successfully assembled."
+                )
+            else:
+                # Coming up with artificial data for now
+                df["cpu_latency"] = (
+                    df["tsp_estimated_e2e_latency"] + df["gpu_e2e_latency"]
+                ) * 10
+                df["tsp_cpu_compute_ratio"] = (
+                    df["cpu_latency"] / df["tsp_estimated_e2e_latency"]
+                )
+                df["gpu_cpu_compute_ratio"] = df["cpu_latency"] / df["gpu_e2e_latency"]
+                data = [
+                    go.Bar(
+                        x=df["model_name"],
+                        y=df["gpu_cpu_compute_ratio"],
+                        name="NVIDIA A100-PCIE-40GB",
+                    ),
+                    go.Bar(
+                        x=df["model_name"],
+                        y=df["tsp_cpu_compute_ratio"],
+                        name="GroqChip 1",
+                    ),
+                    go.Bar(
+                        x=df["model_name"],
+                        y=df["cpu_latency"] * 0 + 1,
+                        name="Intel(R) Xeon(R) Gold 6338 CPU",
+                    ),
+                ]
 
-            fig = dict(data=data, layout=layout)
-            st.plotly_chart(fig, use_container_width=True)
+                layout = go.Layout(
+                    barmode="overlay",  # group
+                    legend={
+                        "orientation": "h",
+                        "xanchor": "center",
+                        "x": 0.5,
+                        "y": 1.2,
+                    },
+                    yaxis_title="Latency Speedup",
+                    colorway=[colorway[2], colorway[1], colorway[0]],
+                    height=600,
+                )
+
+                fig = dict(data=data, layout=layout)
+                st.plotly_chart(fig, use_container_width=True)
+
+                st.markdown(
+                    "<sup>*</sup>Estimated I/O does NOT include delays caused by Groq's runtime.",
+                    unsafe_allow_html=True,
+                )
+                st.markdown(
+                    "<sup>†</sup>Baseline corresponds to Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz.",
+                    unsafe_allow_html=True,
+                )
+
+                with cols[1]:
+                    # Show stats
+                    st.markdown(
+                        f"""<br><br><br><br>
+                        <p style="font-family:sans-serif; font-size: 20px;text-align: center;">Intel(R) Xeon(R) Gold 6338 CPU @ 2.00GHz Acceleration:</p>
+                        <p style="font-family:sans-serif; color:{colorway[0]}; font-size: 26px;text-align: center;"> {1}x (Baseline)</p>
+                        <br><br>
+                        <p style="font-family:sans-serif; font-size: 20px;text-align: center;">NVIDIA A100-PCIE-40GB Acceleration:</p>
+                        <p style="font-family:sans-serif; color:{colorway[2]}; font-size: 26px;text-align: center;"> {round(df["gpu_cpu_compute_ratio"].mean(),2)}x</p>
+                        <p style="font-family:sans-serif; color:{colorway[2]}; font-size: 20px;text-align: center;"> min {round(df["gpu_cpu_compute_ratio"].min(),2)}x; max {round(df["gpu_cpu_compute_ratio"].max(),2)}x</p>
+                        <br><br>
+                        <p style="font-family:sans-serif; font-size: 20px;text-align: center;">GroqChip 1 Acceleration<sup>*</sup>:</p>
+                        <p style="font-family:sans-serif; color:{colorway[1]}; font-size: 26px;text-align: center;"> {round(df["tsp_cpu_compute_ratio"].mean(),2)}x</p>
+                        <p style="font-family:sans-serif; color:{colorway[1]}; font-size: 20px;text-align: center;"> min {round(df["tsp_cpu_compute_ratio"].min(),2)}x; max {round(df["tsp_cpu_compute_ratio"].max(),2)}x</p>""",
+                        unsafe_allow_html=True,
+                    )
+
+    # FAQ Block
+    cols = st.columns(2)
+    with cols[0]:
 
-            st.markdown(
-                "<sup>*</sup>Estimated I/O does NOT include delays caused by Groq's runtime.",
-                unsafe_allow_html=True,
-            )
+        st.markdown(
+            """<style>
+        .big-font {
+            font-size:20px !important;
+        }
+        </style>
+        """,
+            unsafe_allow_html=True,
+        )
 
-        with cols[1]:
-            # Show stats
-            st.markdown(
-                f"""<br><br><br><br><br><br>
-                <p style="font-family:sans-serif; font-size: 20px;text-align: center;">Average speedup of GroqChip™ considering compute only:</p>
-                <p style="font-family:sans-serif; color:#3366cc; font-size: 26px;text-align: center;"> {round(df["tsp_gpu_compute_ratio"].mean(),2)}x</p>
-                <p style="font-family:sans-serif; color:#3366cc; font-size: 20px;text-align: center;"> min {round(df["tsp_gpu_compute_ratio"].min(),2)}x; max {round(df["tsp_gpu_compute_ratio"].max(),2)}x</p>
-                <br><br>
-                <p style="font-family:sans-serif; font-size: 20px;text-align: center;">Average speedup of GroqChip™ considering compute + estimated I/O<sup>*</sup>:</p>
-                <p style="font-family:sans-serif; color:#FF7F0E; font-size: 26px;text-align: center;"> {round(df["tsp_gpu_e2e_ratio"].mean(),2)}x</p>
-                <p style="font-family:sans-serif; color:#FF7F0E; font-size: 20px;text-align: center;"> min {round(df["tsp_gpu_e2e_ratio"].min(),2)}x; max {round(df["tsp_gpu_e2e_ratio"].max(),2)}x</p>""",
-                unsafe_allow_html=True,
-            )
+        class Collapsable:
+            def __init__(self, preamble="", epilogue=""):
+                self.preamble = preamble
+                self.epilogue = epilogue
+                self.sections = []
+
+            def add_section(self, heading, text):
+                self.sections.append((heading, text))
+
+            def deploy(self):
+                small_font = 18
+                large_font = 18
+                secs = "".join(
+                    [
+                        (
+                            f"<details><summary style='font-size:{large_font}px;'>{heading}</summary>"
+                            f"<blockquote><details><summary style='font-size:{small_font}px;max-width: 80%;'>{text}</summary>"
+                            f"<blockquote></blockquote></details></blockquote></details>"
+                        )
+                        for heading, text in self.sections
+                    ]
+                )
+                collapsable_sec = f"""
+                <ol>
+                {self.preamble}
+                {secs}
+                {self.epilogue}
+                </ol>
+                """
+                st.markdown(collapsable_sec, unsafe_allow_html=True)
+
+        st.markdown("""## About this workload analysis (FAQ)""")
+        faq = Collapsable()
+        faq.add_section(
+            "Model selection",
+            'The models that are part of the "ML Agility" set are models that have been internally selected and represent a mix between popular open-source models and models that Groq has historically focused some efforts on (like GNNs).',
+        )
+        faq.add_section(
+            "Experimental Setup",
+            "-",
+        )
+        faq.add_section(
+            "Key limitations",
+            "This set of workloads does not include models with more than 1B parametes.",
+        )
+
+        faq.deploy()
+        st.markdown(
+            "For more details, please visit [mlagility.org](mlagility.org).",
+            unsafe_allow_html=True,
+        )
+
+    st.markdown("## Detailed Data View")
 
-    st.markdown("### Detailed Data View")
-    st.markdown(
-        "**Model selection**: All workloads were obtained from models cards available at huggingface.co/models. Input shapes corresponds exactly to those used by the Huggingface model cards. Some of those input shapes might be small, causing the compilation process to be easier than when reasonably-sized input shapes are used.",
-        unsafe_allow_html=True,
-    )
     model_name = st.text_input("", placeholder="Filter model by name")
     if model_name != "":
         mla_report = mla_report[[model_name in x for x in mla_report["model_name"]]]
 
-    # Select which columns to show
-    selected_cols = list(mla_report.columns)
-    # remove_cols = (
-    #    "tsp_e2e_latency",
-    #    "gpu_e2e_latency",
-    #    "tsp_gpu_e2e_ratio",
-    # )
-    # for item in remove_cols:
-    #    if item in selected_cols:
-    #        selected_cols.remove(item)
+    # Add columns that do not exist yet
+    mla_report["chips_used_gpu"] = 1
+    mla_report["cpu_latency"] = 0
+    mla_report["chips_used_cpu"] = 0
+
+    # Using 2 significant digits
+    mla_report["tsp_estimated_e2e_latency"] = [
+        "-" if x == "-" else "{:.3f}".format(float(x))
+        for x in mla_report["tsp_estimated_e2e_latency"]
+    ]
+    mla_report["gpu_e2e_latency"] = [
+        "-" if x == "-" else "{:.3f}".format(float(x))
+        for x in mla_report["gpu_e2e_latency"]
+    ]
+
+    renamed_cols = {
+        "model_name": "Model Name",
+        "author": "Source",
+        "params": "Parameters",
+        "model_type": "Framework",
+        "tsp_estimated_e2e_latency": "GroqChip 1: Latency (ms)",
+        "gpu_e2e_latency": "NVIDIA A100-PCIE-40GB: Latency (ms)",
+        "cpu_latency": "Intel(R) Xeon(R) Gold 6338 CPU: Latency (ms)",
+        "chips_used": "GroqChip 1: Chips Used",
+        "chips_used_gpu": "NVIDIA A100-PCIE-40GB: Chips Used",
+        "chips_used_cpu": "Intel(R) Xeon(R) Gold 6338 CPU: Chips Used",
+    }
+    mla_report.rename(columns=renamed_cols, inplace=True)
+    selected_cols = renamed_cols.values()
+
     st.dataframe(
-        mla_report[selected_cols], height=min((len(mla_report) + 1) * 35, 35 * 21)
+        mla_report[selected_cols],
+        height=min((len(mla_report) + 1) * 35, 35 * 21),
+        use_container_width=True,
     )

app2.py

@@ -0,0 +1,388 @@
+import time  # to simulate a real time data, time loop
+from os import listdir
+from os.path import isfile, join
+import numpy as np  # np mean, np random
+import pandas as pd  # read csv, df manipulation
+import plotly.express as px  # interactive charts
+from plotly import graph_objs as go
+import streamlit as st  # 🎈 data web app development
+import plotly.figure_factory as ff
+import numpy as np
+from collections import Counter
+
+
+print("Make sure to activate your VPN before running this script")
+
+st.set_page_config(
+    page_title="GroqFlow Progress Tracker",
+    page_icon="🚀",
+    layout="wide",
+)
+
+
+# Session State variables:
+state = st.session_state
+if "INFO_CLOSED" not in state:
+    state.INFO_CLOSED = False
+
+# dashboard title
+st.title("GroqFlow Progress Tracker 🚀")
+
+# Custom chart colors (https://plotly.com/python/discrete-color/)
+colorway = ["#3366cc", "#FF7F0E"]
+
+
+def add_filter(data_frame_list, name, label, options, num_cols=1):
+    st.markdown(f"#### {name}")
+
+    cols = st.columns(num_cols)
+    instantiated_checkbox = []
+    for idx in range(len(options)):
+        with cols[idx % num_cols]:
+            instantiated_checkbox.append(st.checkbox(options[idx], False))
+
+    all_options = set(data_frame_list[-1][label])
+    selected_options = [
+        options[idx] for idx, checked in enumerate(instantiated_checkbox) if checked
+    ]
+
+    # The last checkbox will always correspond to "other"
+    if instantiated_checkbox[-1]:
+        selected_options = selected_options[:-1]
+        other_options = [x for x in all_options if x not in options]
+        selected_options = set(selected_options + other_options)
+
+    if len(selected_options) > 0:
+        for idx in range(len(data_frame_list)):
+            data_frame_list[idx] = data_frame_list[idx][
+                [
+                    any([x == model_entry for x in selected_options])
+                    for model_entry in data_frame_list[idx][label]
+                ]
+            ]
+    return data_frame_list
+
+
+with st.sidebar:
+
+    st.markdown("# Filters")
+
+    test_type = st.radio(
+        "Test Type",
+        ("Daily Tests (100 models)", "Monthly Tests (500+ models)"),
+    )
+    if test_type == "Daily Tests (100 models)":
+        selected_test_type = "daily"
+        report_folder = "reports/daily"
+    else:
+        selected_test_type = "monthly"
+        report_folder = "reports/monthly"
+
+    # Get ML Agility reports
+    reports = sorted(
+        [f for f in listdir(report_folder) if isfile(join(report_folder, f))]
+    )
+
+    selected_report = st.selectbox("Test date", reports, index=len(reports) - 1)
+    selected_report_idx = reports.index(selected_report)
+    prev_report = reports[max(0, selected_report_idx - 1)]
+    mla_report = pd.read_csv(f"{report_folder}/{selected_report}")
+    prev_mla_report = pd.read_csv(f"{report_folder}/{prev_report}")
+
+    # Add chips filter
+    num_chips_options = ["1", "2", "4", "8", "16", "32+"]
+    mla_report = mla_report.astype({"chips_used": str})
+    prev_mla_report = prev_mla_report.astype({"chips_used": str})
+    mla_report, prev_mla_report = add_filter(
+        [mla_report, prev_mla_report],
+        "Number of GroqChips™",
+        label="chips_used",
+        options=num_chips_options,
+        num_cols=3,
+    )
+
+    # Add author filter
+    authors = [
+        "google",
+        "apple",
+        "facebook",
+        "openai",
+        "microsoft",
+        "huggingface",
+        "CompVis",
+        "others",
+    ]
+    mla_report, prev_mla_report = add_filter(
+        [mla_report, prev_mla_report],
+        "Authors",
+        label="author",
+        options=authors,
+        num_cols=2,
+    )
+
+    # Add task filter
+    tasks = [
+        "Image Classification",
+        "Translation",
+        "Image Segmentation",
+        "Fill-Mask",
+        "Text-to-Image",
+        "Token Classification",
+        "Sentence Similarity",
+        "Audio Classification",
+        "Question Answering",
+        "Summarization",
+        "other",
+    ]
+    mla_report, prev_mla_report = add_filter(
+        [mla_report, prev_mla_report], "Tasks", label="task", options=tasks
+    )
+
+
+def detailed_progress_list(df_new, df_old, filter=None):
+    return
+    """
+    if filter is not None:
+        df_new = df_new[(df_new[filter] == True)]
+        df_old = df_old[(df_old[filter] == True)]
+
+    progress = df_new[~(df_new["hash"].isin(df_old["hash"]))].reset_index(drop=True)
+    regression = df_old[~(df_old["hash"].isin(df_new["hash"]))].reset_index(drop=True)
+
+    for model_name in progress["model_name"]:
+        st.markdown(
+            f'<span style="color:green">↑ {model_name}</span>',
+            unsafe_allow_html=True,
+        )
+    for model_name in regression["model_name"]:
+        st.markdown(
+            f'<span style="color:red">↓ {model_name}</span>',
+            unsafe_allow_html=True,
+        )
+    """
+
+
+# creating a single-element container
+placeholder = st.empty()
+
+with placeholder.container():
+
+    st.markdown("## Summary Results")
+    # create three columns
+    kpi = st.columns(7)
+    model_details = st.columns(7)
+
+    # fill in those three columns with respective metrics or KPIs
+    kpi[0].metric(
+        label="All models",
+        value=len(mla_report),
+        delta=len(mla_report) - len(prev_mla_report),
+    )
+    if selected_test_type == "daily":
+        with model_details[0]:
+            detailed_progress_list(mla_report, prev_mla_report)
+
+    kpi[1].metric(
+        label="Convert to ONNX",
+        value=np.sum(mla_report["base_onnx"]),
+        delta=int(
+            np.sum(mla_report["base_onnx"]) - np.sum(prev_mla_report["base_onnx"])
+        ),
+    )
+    if selected_test_type == "daily":
+        with model_details[1]:
+            detailed_progress_list(mla_report, prev_mla_report, "base_onnx")
+
+    kpi[2].metric(
+        label="Optimize ONNX file",
+        value=np.sum(mla_report["optimized_onnx"]),
+        delta=int(
+            np.sum(mla_report["optimized_onnx"])
+            - np.sum(prev_mla_report["optimized_onnx"])
+        ),
+    )
+    if selected_test_type == "daily":
+        with model_details[2]:
+            detailed_progress_list(mla_report, prev_mla_report, "optimized_onnx")
+
+    kpi[3].metric(
+        label="All ops supported",
+        value=np.sum(mla_report["all_ops_supported"]),
+        delta=int(
+            np.sum(mla_report["all_ops_supported"])
+            - np.sum(prev_mla_report["all_ops_supported"])
+        ),
+    )
+    if selected_test_type == "daily":
+        with model_details[3]:
+            detailed_progress_list(mla_report, prev_mla_report, "all_ops_supported")
+
+    kpi[4].metric(
+        label="Converts to FP16",
+        value=np.sum(mla_report["fp16_onnx"]),
+        delta=int(
+            np.sum(mla_report["fp16_onnx"]) - np.sum(prev_mla_report["fp16_onnx"])
+        ),
+    )
+    if selected_test_type == "daily":
+        with model_details[4]:
+            detailed_progress_list(mla_report, prev_mla_report, "fp16_onnx")
+
+    kpi[5].metric(
+        label="Compiles",
+        value=np.sum(mla_report["compiles"]),
+        delta=int(np.sum(mla_report["compiles"]) - np.sum(prev_mla_report["compiles"])),
+    )
+    if selected_test_type == "daily":
+        with model_details[5]:
+            detailed_progress_list(mla_report, prev_mla_report, "compiles")
+
+    kpi[6].metric(
+        label="Assembles",
+        value=np.sum(mla_report["assembles"]),
+        delta=int(
+            np.sum(mla_report["assembles"]) - np.sum(prev_mla_report["assembles"])
+        ),
+    )
+    if selected_test_type == "daily":
+        with model_details[6]:
+            detailed_progress_list(mla_report, prev_mla_report, "assembles")
+
+    cols = st.columns(2)
+    with cols[0]:
+
+        compiler_errors = mla_report[mla_report["compiler_error"] != "-"][
+            "compiler_error"
+        ]
+        compiler_errors = Counter(compiler_errors)
+        st.markdown("""#### Top compiler issues""")
+        if len(compiler_errors) > 0:
+            compiler_errors = pd.DataFrame.from_dict(
+                compiler_errors, orient="index"
+            ).reset_index()
+            compiler_errors = compiler_errors.set_axis(
+                ["error", "count"], axis=1, inplace=False
+            )
+
+            fig = px.bar(
+                compiler_errors, x="count", y="error", orientation="h", height=400
+            )
+            st.plotly_chart(fig, use_container_width=True)
+        else:
+            st.markdown("""No compiler errors found :tada:""")
+
+    with cols[1]:
+        # Add parameters histogram
+        all_models = [float(x) / 1000000 for x in mla_report["params"] if x != "-"]
+
+        assembled_models = mla_report[mla_report["assembles"] == True]
+        assembled_models = [
+            float(x) / 1000000 for x in assembled_models["params"] if x != "-"
+        ]
+        hist_data = []
+        group_labels = []
+        if all_models != []:
+            hist_data.append(all_models)
+            group_labels.append("Models we tried compiling")
+
+        if assembled_models != []:
+            hist_data.append(assembled_models)
+            group_labels.append("Assembled models")
+
+        st.markdown("""#### Assembled models vs. Parameters (in millions)""")
+
+        if len(assembled_models) > 1:
+
+            fig = ff.create_distplot(
+                hist_data,
+                group_labels,
+                bin_size=[25, 25],
+                histnorm="",
+            )
+            # fig.layout.update(title="Assembled models vs. Parameters (in millions)")
+            fig.layout.update(xaxis_title="Parameters in millions")
+            fig.layout.update(yaxis_title="count")
+            fig.update_xaxes(range=[1, 1000])
+            st.plotly_chart(fig, use_container_width=True)
+        else:
+            st.markdown("""Need at least one assembled model to show this graph 😅""")
+
+    if "tsp_gpu_compute_ratio" in mla_report and "tsp_gpu_e2e_ratio" in mla_report:
+        cols = st.columns(2)
+        with cols[0]:
+            # GPU Acceleration plot
+            st.markdown("""#### Speedup of GroqChip™ compared to A100 GPUs""")
+
+            # Prepare data
+            df = mla_report[
+                ["model_name", "tsp_gpu_compute_ratio", "tsp_gpu_e2e_ratio"]
+            ]
+            df = df.sort_values(by=["model_name"])
+            df = df[(df.tsp_gpu_compute_ratio != "-")]
+            df = df[(df.tsp_gpu_e2e_ratio != "-")]
+            df["tsp_gpu_compute_ratio"] = df["tsp_gpu_compute_ratio"].astype(float)
+            df["tsp_gpu_e2e_ratio"] = df["tsp_gpu_e2e_ratio"].astype(float)
+
+            data = [
+                go.Bar(
+                    x=df["model_name"],
+                    y=df["tsp_gpu_compute_ratio"],
+                    name="Compute only",
+                ),
+                go.Bar(
+                    x=df["model_name"],
+                    y=df["tsp_gpu_e2e_ratio"],
+                    name="Compute + estimated I/O",
+                ),
+            ]
+
+            layout = go.Layout(
+                barmode="overlay",
+                yaxis_title="Speedup compared to A100 GPU",
+                colorway=colorway,
+            )
+
+            fig = dict(data=data, layout=layout)
+            st.plotly_chart(fig, use_container_width=True)
+
+            st.markdown(
+                "<sup>*</sup>Estimated I/O does NOT include delays caused by Groq's runtime.",
+                unsafe_allow_html=True,
+            )
+
+        with cols[1]:
+            # Show stats
+            st.markdown(
+                f"""<br><br><br><br><br><br>
+                <p style="font-family:sans-serif; font-size: 20px;text-align: center;">Average speedup of GroqChip™ considering compute only:</p>
+                <p style="font-family:sans-serif; color:#3366cc; font-size: 26px;text-align: center;"> {round(df["tsp_gpu_compute_ratio"].mean(),2)}x</p>
+                <p style="font-family:sans-serif; color:#3366cc; font-size: 20px;text-align: center;"> min {round(df["tsp_gpu_compute_ratio"].min(),2)}x; max {round(df["tsp_gpu_compute_ratio"].max(),2)}x</p>
+                <br><br>
+                <p style="font-family:sans-serif; font-size: 20px;text-align: center;">Average speedup of GroqChip™ considering compute + estimated I/O<sup>*</sup>:</p>
+                <p style="font-family:sans-serif; color:#FF7F0E; font-size: 26px;text-align: center;"> {round(df["tsp_gpu_e2e_ratio"].mean(),2)}x</p>
+                <p style="font-family:sans-serif; color:#FF7F0E; font-size: 20px;text-align: center;"> min {round(df["tsp_gpu_e2e_ratio"].min(),2)}x; max {round(df["tsp_gpu_e2e_ratio"].max(),2)}x</p>""",
+                unsafe_allow_html=True,
+            )
+
+    st.markdown("### Detailed Data View")
+    st.markdown(
+        "**Model selection**: All workloads were obtained from models cards available at huggingface.co/models. Input shapes corresponds exactly to those used by the Huggingface model cards. Some of those input shapes might be small, causing the compilation process to be easier than when reasonably-sized input shapes are used.",
+        unsafe_allow_html=True,
+    )
+    model_name = st.text_input("", placeholder="Filter model by name")
+    if model_name != "":
+        mla_report = mla_report[[model_name in x for x in mla_report["model_name"]]]
+
+    # Select which columns to show
+    selected_cols = list(mla_report.columns)
+    # remove_cols = (
+    #    "tsp_e2e_latency",
+    #    "gpu_e2e_latency",
+    #    "tsp_gpu_e2e_ratio",
+    # )
+    # for item in remove_cols:
+    #    if item in selected_cols:
+    #        selected_cols.remove(item)
+    st.dataframe(
+        mla_report[selected_cols], height=min((len(mla_report) + 1) * 35, 35 * 21)
+    )

reports/mlagility/2023-01-09.csv

@@ -0,0 +1,260 @@
+model_name,author,class,downloads,base_onnx,optimized_onnx,all_ops_supported,fp16_onnx,compiles,assembles,params,chips_used,hash,license,task,model_type,cycles,tsp_compute_latency,gpu_compute_latency,tsp_gpu_compute_ratio,tsp_estimated_e2e_latency,gpu_e2e_latency,tsp_gpu_e2e_ratio,compiler_error,export_time,optimize_onnx_time,check_compatibility_time,fp16_conversion_time,compile_time,assemble_time,compiler_ram_GB
+albert base v1,huggingface tf,TFAlbertModel,0,True,True,True,True,True,True,11623453,1,d6b7568a,-,-,keras,422717,0.4696855555555556,0.84697,1.8032702730195378,0.49796288888888895,0.878904,1.7649989981404235,-,51.5570330619812,0.9572975635528564,6.393486738204956,1.3821609020233154,547.0339939594269,759.8349900245667,10.0
+albert base v2,huggingface tf,TFAlbertModel,0,True,True,True,True,True,True,11623455,1,d6b7568a,-,-,keras,422554,0.46950444444444445,0.845832,1.801542051430113,0.4977817777777778,0.877898,1.7636202030519395,-,51.02895498275757,1.01800537109375,5.380262613296509,1.4903111457824707,542.994854927063,773.7713937759399,10.0
+albert large v1,huggingface tf,TFAlbertModel,0,True,True,True,True,False,False,17620253,1,248c46e7,-,-,keras,-,-,-,-,-,-,-,error: 'groq.alloc' op scheduleOp failed,152.56399726867676,1.7646169662475586,5.5928566455841064,2.043565273284912,0,0,0.0
+albert large v2,huggingface tf,TFAlbertModel,0,True,True,True,True,False,False,17620255,1,248c46e7,-,-,keras,-,-,-,-,-,-,-,error: 'groq.alloc' op scheduleOp failed,153.047536611557,2.000840187072754,7.1911962032318115,2.26590633392334,0,0,0.0
+alexnet,torch hub vision,AlexNet,0,True,True,True,True,True,True,61100840,2,2891f54c,-,-,pytorch,47442,0.10542666666666667,0.237213,2.2500284557986596,0.138054,0.278432,2.01683399249569,-,4.0610644817352295,2.9978034496307373,3.518373966217041,6.550444841384888,153.59116911888123,67.59902596473694,2.0
+bart,huggingface,BartModel,0,True,True,True,True,False,False,404078238,16,cb0751ce,-,-,pytorch,-,-,-,-,-,-,-,Groq Compiler exited,70.96846866607666,20.3438138961792,8.69025993347168,42.1454222202301,0,0,0.0
+beit,huggingface,BeitModel,0,True,True,True,True,True,False,85530736,2,6b5d54c6,-,-,pytorch,308388,0.6853066666666666,-,-,-,-,-,-,11.883913278579712,4.567528009414673,5.666049003601074,8.541035890579224,824.6184940338135,0,15.0
+bert base cased,huggingface tf,TFBertModel,0,True,True,True,True,True,True,107991579,4,87d9339a,-,-,keras,240704,1.0697955555555556,0.871612,0.8147463274395108,1.0980728888888889,0.903687,0.8229754228013199,-,64.88645625114441,9.298788070678711,5.868211030960083,12.375629901885986,677.0152542591095,633.2466506958008,11.0
+bert base cased finetuned mrpc,huggingface tf,TFBertModel,0,True,True,True,True,True,True,107991579,4,87d9339a,-,-,keras,240704,1.0697955555555556,0.853524,0.7978384239563945,1.0980728888888889,0.88561,0.8065129455077663,-,61.80271124839783,7.134819507598877,6.987076997756958,12.610964298248291,646.635950088501,538.9989869594574,11.0
+bert base chinese,huggingface tf,TFBertModel,0,True,True,True,True,True,True,101948955,4,af9e53c1,-,-,keras,234311,1.0413822222222222,0.863168,0.8288676161170411,1.0696595555555555,0.895492,0.8371747771045741,-,64.96016240119934,5.8003222942352295,5.400928974151611,11.991683006286621,662.6369862556458,619.843807220459,11.0
+bert base german cased,huggingface tf,TFBertModel,0,True,True,True,True,True,True,108762651,4,3df992fb,-,-,keras,240137,1.0672755555555555,0.875198,0.8200300245276655,1.0955528888888888,0.907695,0.8285268645684332,-,63.41416072845459,5.662364482879639,6.544678688049316,12.178314208984375,656.9293267726898,616.582097530365,11.0
+bert base multilingual cased,huggingface tf,TFBertModel,0,True,True,True,True,False,False,177534747,4,ae4a36ca,-,-,keras,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,82.13504695892334,10.183184385299683,7.17675256729126,18.873542070388794,0,0,3.0
+bert base multilingual uncased,huggingface tf,TFBertModel,0,True,True,True,True,False,False,167037723,4,80b7b795,-,-,keras,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,78.95565962791443,8.319761276245117,19.111674785614014,19.968615531921387,0,0,3.0
+bert base uncased,huggingface tf,TFBertModel,0,True,True,True,True,True,True,109163547,4,734b2447,-,-,keras,240687,1.06972,0.87003,0.8133249822383427,1.0979973333333333,0.902069,0.8215584615870348,-,61.262481689453125,5.941906690597534,5.069867372512817,16.26664161682129,656.9287917613983,566.3101632595062,11.0
+bert,huggingface,BertModel,0,True,True,True,True,True,True,109166702,4,d59172a2,-,-,pytorch,322992,1.43552,0.860402,0.599366083370486,1.463808,0.895693,0.6118924066544246,-,13.090381860733032,5.825865268707275,6.316922187805176,12.856344938278198,686.6346187591553,652.7709035873413,12.0
+bert for question answering,huggingface,BertForQuestionAnswering,0,True,True,True,True,True,False,333701331,8,64bce7df,-,-,pytorch,413079,3.671813333333333,-,-,-,-,-,-,39.125213384628296,17.447813272476196,7.8959879875183105,36.06975960731506,2072.01473236084,0,34.0
+bert generation,huggingface,EncoderDecoderModel,0,True,True,True,True,False,False,465654029,16,c8f4fe85,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,218.3731348514557,33.44469213485718,19.55678677558899,62.489097595214844,0,0,3.0
+bert large cased,huggingface tf,TFBertModel,0,True,True,True,True,True,False,332994587,8,28edf212,-,-,keras,486329,4.322924444444444,-,-,-,-,-,-,197.78413152694702,19.80856442451477,7.942766189575195,36.79043364524841,2111.486341238022,0,35.0
+bert large cased whole word masking,huggingface tf,TFBertModel,0,True,True,True,True,True,False,332994587,8,28edf212,-,-,keras,486329,4.322924444444444,-,-,-,-,-,-,200.06373405456543,17.3145694732666,8.489120483398438,32.96453499794006,2077.9071395397186,0,35.0
+bert large cased whole word masking finetuned squad,huggingface tf,TFBertModel,0,True,True,True,True,True,False,332994587,8,28edf212,-,-,keras,486329,4.322924444444444,-,-,-,-,-,-,197.327232837677,21.96155881881714,7.155290603637695,36.937096118927,1918.883693933487,0,35.0
+bert large uncased,huggingface tf,TFBertModel,0,True,True,True,True,True,False,334557211,8,433eed41,-,-,keras,486329,4.322924444444444,-,-,-,-,-,-,207.92871856689453,18.454949855804443,16.44332194328308,41.77619552612305,2080.774171113968,0,35.0
+bert large uncased whole word masking,huggingface tf,TFBertModel,0,True,True,True,True,True,False,334557211,8,433eed41,-,-,keras,486329,4.322924444444444,-,-,-,-,-,-,197.6210424900055,17.639880418777466,8.113890171051025,38.771852016448975,2048.3606622219086,0,35.0
+bert large uncased whole word masking finetuned squad,huggingface tf,TFBertModel,0,True,True,True,True,True,False,334557211,8,433eed41,-,-,keras,486329,4.322924444444444,-,-,-,-,-,-,187.9991855621338,15.494677782058716,7.9293365478515625,33.16516137123108,1868.4096915721893,0,35.0
+bert tiny for sequence classification,huggingface,BertForSequenceClassification,0,True,True,True,True,True,True,4353184,1,ca662a9e,-,-,pytorch,27023,0.030025555555555555,0.10016,3.3358250379306518,0.05005238888888889,0.120791,2.4132914068925557,-,1.0278284549713135,0.5196475982666016,5.6260459423065186,0.5758223533630371,49.74038338661194,24.64625883102417,0.73828125
+blenderbot small,huggingface,BlenderbotSmallModel,0,True,True,True,True,True,False,84606649,2,d65dd9e3,-,-,pytorch,1517482,3.3721822222222224,-,-,-,-,-,-,21.777433395385742,4.6429126262664795,4.678505182266235,9.720015525817871,1051.1570615768433,0,23.0
+camembert,huggingface,CamembertModel,0,True,True,True,True,True,True,109461617,4,3e856449,-,-,pytorch,269308,1.1969244444444445,1.15116,0.9617649679920388,1.2252124444444443,1.18639,0.9683137037821651,-,16.99555253982544,5.350171804428101,6.712750434875488,11.690237760543823,671.3604502677917,616.2100164890289,11.0
+cl tohoku bert base japanese,huggingface tf,TFBertModel,0,True,True,True,True,True,True,110298651,4,62e15052,-,-,keras,242513,1.0778355555555557,0.85116,0.7896937483763756,1.106112888888889,0.883185,0.7984582847481109,-,63.151143074035645,5.702507019042969,6.581105470657349,11.4367196559906,618.8648188114166,540.9790978431702,11.0
+cl tohoku bert base japanese char,huggingface tf,TFBertModel,0,True,True,True,True,True,True,88794651,2,e05d78b5,-,-,keras,244257,0.5427933333333333,0.873801,1.6098226458197717,0.5710706666666666,0.905748,1.5860523974849583,-,61.4022536277771,4.68958592414856,17.556321144104004,10.806075811386108,637.3437783718109,664.2115099430084,10.0
+cl tohoku bert base japanese char whole word masking,huggingface tf,TFBertModel,0,True,True,True,True,True,True,88794651,2,e05d78b5,-,-,keras,244257,0.5427933333333333,0.872969,1.608289834068215,0.5710706666666666,0.90502,1.5847775990361965,-,57.316322565078735,4.424041271209717,6.797808647155762,8.594735860824585,565.547210931778,619.0271394252777,10.0
+cl tohoku bert base japanese whole word masking,huggingface tf,TFBertModel,0,True,True,True,True,True,True,110298651,4,62e15052,-,-,keras,242513,1.0778355555555557,0.851619,0.7901196018357778,1.106112888888889,0.883634,0.7988642107657085,-,70.79523658752441,7.035403490066528,6.276620149612427,15.840601682662964,678.8624987602234,576.5854253768921,11.0
+clip text encoder,stable diffusion,CLIPTextModel,0,True,True,True,True,True,True,123066514,4,d312ecd1,-,-,pytorch,230570,1.0247555555555556,-,-,1.049760388888889,-,-,-,16.220991849899292,6.211384057998657,5.673836946487427,13.057928085327148,530.6008114814758,389.18997859954834,8.0
+convbert,huggingface,ConvBertModel,0,True,True,True,True,False,False,105388842,4,b39013e9,-,-,pytorch,-,-,-,-,-,-,-,Compiler log is empty,18.855659008026123,5.670386791229248,6.077420711517334,11.364919662475586,0,0,0.0
+convnext,huggingface,ConvNextModel,0,True,True,True,True,False,False,27766372,1,80414def,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,4.822420597076416,1.7179253101348877,4.550223350524902,3.370676040649414,0,0,0.0
+convnext base,torch hub vision,ConvNeXt,0,True,True,True,True,False,False,88438950,2,bcaefd44,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,11.923899173736572,4.660487413406372,5.864811658859253,8.821153163909912,0,0,0.0
+convnext large,torch hub vision,ConvNeXt,0,True,True,True,True,False,False,197538470,4,6500d01c,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,19.084625482559204,10.089269399642944,7.093344449996948,20.197072982788086,0,0,1.0
+convnext small,torch hub vision,ConvNeXt,0,True,True,True,True,False,False,50109350,2,b12ad476,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,8.499674558639526,2.8249852657318115,4.821101903915405,5.204307556152344,0,0,0.0
+convnext tiny,torch hub vision,ConvNeXt,0,True,True,True,True,False,False,28536908,1,6ff16bbc,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,4.1630964279174805,1.557394027709961,5.346322059631348,2.8586907386779785,0,0,0.0
+ctrl,huggingface tf,TFCTRLModel,0,False,False,False,False,False,False,-,-,3c2b5ffc,-,-,keras,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+deberta,huggingface,DebertaModel,0,True,True,True,True,True,True,123642074,4,f4e4f0d1,-,-,pytorch,337669,1.5007511111111111,0.910708,0.6068347997595278,1.5289751111111112,0.944085,0.6174626343746893,-,28.8400936126709,6.527955532073975,5.610030651092529,12.73327088356018,738.6040179729462,654.0242984294891,12.0
+deit,huggingface,DeiTModel,0,True,True,True,True,True,False,86272621,2,4519cd75,-,-,pytorch,326836,0.7263022222222222,-,-,-,-,-,-,11.146126747131348,4.565266847610474,5.67544150352478,9.194467306137085,797.3013422489166,0,15.0
+deit base for image classification,huggingface,ViTForImageClassification,0,True,True,True,True,True,False,86567765,2,8fa842d1,-,-,pytorch,320526,0.71228,-,-,-,-,-,-,11.148566246032715,4.8765623569488525,5.946186542510986,8.882615327835083,851.1567351818085,0,15.0
+deit tiny for image classification,huggingface,ViTForImageClassification,0,True,True,True,True,True,True,5717525,1,4f7bba18,-,-,pytorch,147721,0.16413444444444444,0.785855,4.787873762024357,0.19676177777777776,0.82644,4.200206002069056,-,4.446189641952515,0.4978640079498291,4.361860990524292,0.7343482971191406,175.61279344558716,163.92737865447998,3.0
+densenet121,torch hub vision,DenseNet,0,True,True,True,True,False,False,7928960,1,d5f7254d,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,14.067578792572021,0.7201809883117676,5.678577661514282,1.1396300792694092,0,0,0.0
+densenet161,torch hub vision,DenseNet,0,True,True,True,True,False,False,28564768,1,6c360ce5,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,24.228121757507324,2.973773956298828,3.7153451442718506,3.1227447986602783,0,0,0.0
+densenet169,torch hub vision,DenseNet,0,True,True,True,True,False,False,14079232,1,ccd997cb,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,25.87918519973755,0.9873182773590088,3.552551507949829,1.7267746925354004,0,0,0.0
+densenet201,torch hub vision,DenseNet,0,True,True,True,True,False,False,19901952,1,e355a66c,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,38.554643392562866,1.240544319152832,9.71229863166809,2.2508044242858887,0,0,0.0
+detr,huggingface,DetrModel,0,True,True,True,False,False,False,-,-,c328f5b8,-,-,pytorch,-,-,-,-,-,-,-,-,33.85950326919556,2.4741523265838623,6.720499753952026,0,0,0,0.0
+detr for object detection,huggingface,DetrForObjectDetection,0,True,True,True,False,False,False,-,-,a2481ba5,-,-,pytorch,-,-,-,-,-,-,-,-,32.385143756866455,3.0429885387420654,6.138234853744507,0,0,0,0.0
+distil wav2vec2 for audio classification,huggingface,Wav2Vec2ForSequenceClassification,0,True,True,True,True,False,False,37866331,1,cd811c97,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,4.754003286361694,1.9989101886749268,6.707224130630493,4.01720404624939,0,0,0.0
+distilbert,huggingface,DistilBertModel,0,True,True,True,True,False,False,66068065,2,38518005,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,6.622883319854736,3.592888116836548,6.078198671340942,7.598631381988525,0,0,0.0
+distilbert base cased,huggingface tf,TFDistilBertModel,0,True,True,True,True,True,True,64836116,2,0910842f,-,-,keras,157400,0.3497777777777778,0.438513,1.2536902795425666,0.3779911111111111,0.468515,1.2394868192079767,-,37.93720269203186,3.7084007263183594,6.097744941711426,7.160471677780151,406.6120777130127,323.9577376842499,6.0
+distilbert base cased distilled squad,huggingface tf,TFDistilBertModel,0,True,True,True,True,True,True,64836116,2,0910842f,-,-,keras,157400,0.3497777777777778,0.441412,1.2619783989834816,0.3779911111111111,0.471535,1.2474764250776031,-,35.37566924095154,3.3954122066497803,5.200200080871582,6.25512957572937,347.463529586792,312.0937433242798,6.0
+distilbert base multilingual cased,huggingface tf,TFDistilBertModel,0,True,True,True,True,False,False,134379284,4,01be3f68,-,-,keras,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,52.42826795578003,6.778413534164429,5.846157073974609,19.83406972885132,0,0,3.0
+distilbert base uncased,huggingface tf,TFDistilBertModel,0,True,True,True,True,True,True,66008084,2,da36addf,-,-,keras,156857,0.3485711111111111,0.430096,1.2338830909682066,0.3767844444444445,0.459979,1.2208014603103454,-,33.650227785110474,3.472047805786133,7.132587432861328,7.806470632553101,364.01527214050293,284.75679206848145,6.0
+distilbert base uncased distilled squad,huggingface tf,TFDistilBertModel,0,True,True,True,True,True,True,66008084,2,da36addf,-,-,keras,156857,0.3485711111111111,0.435178,1.2484626124431808,0.3767844444444445,0.46539,1.2351624565769994,-,41.603827238082886,5.684598684310913,4.957815408706665,6.905161619186401,372.5050690174103,318.56770944595337,6.0
+distilbert base uncased finetuned sst 2 english,huggingface tf,TFDistilBertModel,0,True,True,True,True,True,True,66008084,2,da36addf,-,-,keras,156857,0.3485711111111111,0.433036,1.2423175248793485,0.3767844444444445,0.462934,1.2286441407701425,-,36.073076248168945,3.6800177097320557,5.123138427734375,7.0913777351379395,373.35548734664917,346.0086531639099,6.0
+distilbert for question answering,huggingface,DistilBertForQuestionAnswering,0,True,True,True,True,False,False,66069607,2,65b3ff1b,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,6.255059719085693,3.55653977394104,5.962494611740112,7.072138786315918,0,0,0.0
+distilgpt2,huggingface tf,TFGPT2Model,0,True,True,True,True,True,False,81196570,2,c2eb3fdb,-,-,keras,271880,0.6041777777777778,-,-,-,-,-,-,44.88999319076538,5.3711628913879395,5.991098165512085,9.501911163330078,481.5806887149811,0,8.0
+distilhubert for audio classification,huggingface,HubertForSequenceClassification,0,True,True,True,True,False,False,23700596,1,4170140a,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,2.6910297870635986,1.2741451263427734,5.587473392486572,2.3160693645477295,0,0,0.0
+distilroberta base,huggingface tf,TFRobertaModel,0,True,True,True,True,True,True,82155293,2,3807c8c8,-,-,keras,187088,0.4157511111111111,0.761553,1.8317521701017703,0.44402844444444445,0.793522,1.7870972230007285,-,40.34678769111633,4.274987459182739,5.981934309005737,8.96199893951416,390.15789818763733,341.9048058986664,7.0
+efficientnet b0,torch hub vision,EfficientNet,0,True,True,True,True,True,True,5242196,1,94890704,-,-,pytorch,198515,0.22057222222222223,0.863588,3.915216482381684,0.25319955555555557,0.905288,3.575393321736566,-,5.396315097808838,0.4749176502227783,3.8099679946899414,0.648871898651123,478.253280878067,364.7353434562683,6.0
+efficientnet b1,torch hub vision,EfficientNet,0,True,True,True,True,True,True,7724900,1,8e53a932,-,-,pytorch,304853,0.33872555555555556,1.21577,3.589247932610143,0.3713528888888889,1.25728,3.385674482732208,-,9.503531694412231,0.5952889919281006,3.0240137577056885,0.9774770736694336,787.6342966556549,642.6330316066742,9.0
+efficientnet b2,torch hub vision,EfficientNet,0,True,True,True,True,True,True,9034582,1,204800dc,-,-,pytorch,310259,0.3447322222222222,1.2597,3.65414057287621,0.37735955555555556,1.30121,3.4481967684224535,-,9.44528841972351,0.6639859676361084,3.9450910091400146,1.058732271194458,764.2471086978912,626.4800250530243,10.0
+efficientnet b3,torch hub vision,EfficientNet,0,True,True,True,True,True,True,12134224,1,2950ca5b,-,-,pytorch,316337,0.35148555555555555,1.44895,4.122360014794349,0.3841128888888889,1.49068,3.88083827208205,-,12.284668445587158,0.8280420303344727,3.9773917198181152,1.3564410209655762,664.9719686508179,589.7380015850067,9.0
+efficientnet b4,torch hub vision,EfficientNet,0,True,True,True,True,False,False,19197120,1,7d75dda2,-,-,pytorch,-,-,-,-,-,-,-,error: 'groq.alloc' op scheduleOp failed,18.803763389587402,1.2153074741363525,3.9763386249542236,2.353219747543335,0,0,0.0
+efficientnet b6,torch hub vision,EfficientNet,0,True,True,True,True,False,False,42776110,1,d5bd9458,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,32.90444779396057,2.323835849761963,3.503824472427368,4.556901931762695,0,0,0.8193359375
+efficientnet b7,torch hub vision,EfficientNet,0,True,True,True,True,False,False,65977888,2,6973429a,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,50.130390644073486,3.344799518585205,3.7694854736328125,6.644639730453491,0,0,1.0
+efficientnet v2 l,torch hub vision,EfficientNet,0,True,True,True,True,False,False,117896136,4,f5ddf7f0,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,79.98838019371033,5.6942408084869385,4.090556621551514,11.929809331893921,0,0,3.0
+efficientnet v2 m,torch hub vision,EfficientNet,0,True,True,True,True,False,False,53790556,2,a041aef8,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,48.306984186172485,7.802056550979614,4.9556663036346436,5.562887191772461,0,0,1.0
+efficientnet v2 s,torch hub vision,EfficientNet,0,True,True,True,True,False,False,21275536,1,ae743058,-,-,pytorch,-,-,-,-,-,-,-,error: 'groq.alloc' op scheduleOp failed,20.32683515548706,1.2593109607696533,4.128829002380371,2.279055595397949,0,0,0.0
+electra,huggingface,ElectraModel,0,True,True,True,True,True,True,13411437,1,8da49ae6,-,-,pytorch,141616,0.1573511111111111,0.574369,3.650237967461304,0.18011377777777776,0.597589,3.31784168525574,-,5.198820114135742,1.7192060947418213,5.405793905258179,1.465705394744873,150.8570749759674,126.09572267532349,2.0
+electra for sequence classification,huggingface,ElectraForSequenceClassification,0,True,True,True,True,True,True,109285747,4,5ccb19c4,-,-,pytorch,323950,1.4397777777777778,0.867519,0.6025367340638987,1.4598099444444443,0.886719,0.6074208518544214,-,15.9360032081604,6.086381912231445,6.504514932632446,12.375548601150513,774.8792722225189,642.1030042171478,12.0
+eleutherai gpt j 6b,huggingface tf,TFGPTJModel,0,False,False,False,False,False,False,-,-,87515e19,-,-,keras,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+encoder decoder,huggingface,EncoderDecoderModel,0,True,True,True,True,True,False,269541422,8,051eeb05,-,-,pytorch,1785901,15.874675555555555,-,-,-,-,-,-,44.52228760719299,13.375491380691528,6.7812230587005615,28.296337842941284,2216.6825335025787,0,54.0
+facebook bart large,huggingface tf,TFBartModel,0,True,True,True,True,False,False,404073646,16,7c55159a,-,-,keras,-,-,-,-,-,-,-,Groq Compiler exited,248.80936551094055,29.72284173965454,8.750951528549194,42.18157768249512,0,0,0.0
+facebook opt 350m,huggingface tf,TFOPTModel,0,True,True,True,True,True,False,328926248,8,f58db0fb,-,-,keras,1007716,8.957475555555556,-,-,-,-,-,-,215.54045701026917,17.39953923225403,7.059905767440796,35.949859857559204,2217.7839057445526,0,45.0
+fasterrcnn mobilenet v3 large 320 fpn,torchvision,FasterRCNN,0,True,True,False,False,False,False,-,-,59bcc1a5,-,-,pytorch,-,-,-,-,-,-,-,-,21.89487862586975,1.4812824726104736,0,0,0,0,0.0
+fasterrcnn mobilenet v3 large fpn,torchvision,FasterRCNN,0,True,True,False,False,False,False,-,-,e32c9090,-,-,pytorch,-,-,-,-,-,-,-,-,20.968358516693115,1.5705420970916748,0,0,0,0,0.0
+fasterrcnn resnet50 fpn,torchvision,FasterRCNN,0,True,True,False,False,False,False,-,-,d8b3f65a,-,-,pytorch,-,-,-,-,-,-,-,-,27.44887399673462,2.541313648223877,0,0,0,0,0.0
+fasterrcnn resnet50 fpn v2,torchvision,FasterRCNN,0,True,True,False,False,False,False,-,-,7147702b,-,-,pytorch,-,-,-,-,-,-,-,-,25.006882667541504,2.5054311752319336,0,0,0,0,0.0
+fcos resnet50 fpn,torchvision,FCOS,0,True,True,False,False,False,False,-,-,78b52a80,-,-,pytorch,-,-,-,-,-,-,-,-,35.763006925582886,1.9924123287200928,0,0,0,0,0.0
+flaubert,huggingface,FlaubertModel,0,True,True,True,True,False,False,665991362,16,6202b0cf,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,32.121158838272095,3.511162519454956,9.943092107772827,30.715230464935303,0,0,0.0
+funnel,huggingface,FunnelModel,0,True,True,False,False,False,False,-,-,ab8f5fd3,-,-,pytorch,-,-,-,-,-,-,-,-,41.632219552993774,6.376246213912964,0,0,0,0,0.0
+funnel base,huggingface,FunnelBaseModel,0,True,True,True,True,False,False,111060503,4,37ecc84c,-,-,pytorch,-,-,-,-,-,-,-,[error]	DecomposeONNXToONNXPass failed,31.839351892471313,5.946577072143555,8.634498834609985,11.378712892532349,0,0,1.0
+funnel transformer intermediate base,huggingface tf,TFFunnelBaseModel,0,True,True,True,True,False,False,153551048,4,bdefea98,-,-,keras,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,113.35694766044617,9.043661832809448,6.678390264511108,18.2687828540802,0,0,0.0
+funnel transformer intermediate,huggingface tf,TFFunnelModel,0,True,True,True,True,False,False,168107799,4,f1a953f7,-,-,keras,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,132.02932906150818,14.725075244903564,13.328724384307861,19.625896692276,0,0,0.0
+funnel transformer medium base,huggingface tf,TFFunnelBaseModel,0,True,True,True,True,False,False,110765768,4,842b0bdf,-,-,keras,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,108.40182542800903,7.384202003479004,7.757920503616333,15.319437980651855,0,0,0.0
+funnel transformer medium,huggingface tf,TFFunnelModel,0,True,True,True,True,False,False,125322519,4,ca04688f,-,-,keras,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,125.97902774810791,9.089191198348999,9.433595895767212,13.895658731460571,0,0,0.0
+funnel transformer small base,huggingface tf,TFFunnelBaseModel,0,True,True,True,True,False,False,110372360,4,842b0bdf,-,-,keras,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,83.16907811164856,6.44403338432312,7.304959535598755,13.040780782699585,0,0,0.0
+funnel transformer small,huggingface tf,TFFunnelModel,0,True,True,True,True,False,False,124929111,4,ca04688f,-,-,keras,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,97.88805270195007,6.776267766952515,7.61335825920105,17.184364557266235,0,0,0.0
+google electra base discriminator,huggingface tf,TFElectraModel,0,True,True,True,True,True,True,108573716,4,1cf321ab,-,-,keras,240367,1.0682977777777778,0.855204,0.800529606809587,1.096511111111111,0.885313,0.8073908153132158,-,61.71648097038269,5.7692248821258545,6.0194103717803955,12.372496128082275,678.2114639282227,561.1945736408234,11.0
+google electra base generator,huggingface tf,TFElectraModel,0,True,True,True,True,True,True,33273110,1,a1e97e9b,-,-,keras,162715,0.18079444444444445,0.592178,3.2754214423992867,0.20354644444444445,0.612194,3.007637896456064,-,20.682780504226685,2.0993454456329346,5.079251050949097,3.709099769592285,205.00832557678223,160.4654302597046,4.0
+google electra large discriminator,huggingface tf,TFElectraModel,0,True,True,True,True,True,True,333508628,8,d75e96b1,-,-,keras,478596,4.254186666666667,1.92424,0.45231677657147157,4.285130666666666,1.95915,0.45719726010688744,-,194.6130359172821,17.41837191581726,9.799250602722168,33.113736391067505,1994.2258851528168,1724.4234309196472,34.0
+google electra large generator,huggingface tf,TFElectraModel,0,True,True,True,True,True,True,50655508,2,e262e3ba,-,-,keras,190375,0.4230555555555556,1.1725,2.7715036112935,0.44580755555555557,1.1927,2.6753696413101022,-,35.38599109649658,3.0545120239257812,7.849907398223877,5.587621450424194,380.80264019966125,355.5666253566742,6.0
+google electra small discriminator,huggingface tf,TFElectraModel,0,True,True,True,True,True,True,13393686,1,a0982c45,-,-,keras,147668,0.16407555555555556,0.580635,3.538826963187691,0.18682755555555552,0.600982,3.216773875849863,-,15.661126136779785,1.0657153129577637,4.978039979934692,1.468364953994751,156.9095778465271,135.15213418006897,3.0
+google electra small generator,huggingface tf,TFElectraModel,0,True,True,True,True,True,True,13393686,1,a0982c45,-,-,keras,147902,0.16433555555555557,0.591444,3.599002041892604,0.18708755555555553,0.611657,3.2693622950157626,-,15.98181438446045,0.9070100784301758,4.979582071304321,1.6079082489013672,160.71817541122437,139.64960193634033,3.0
+google mobilebert uncased,huggingface tf,TFMobileBertModel,0,True,True,True,True,True,True,24288553,1,fd77587c,-,-,keras,224772,0.24974666666666667,4.2931,17.18981901660349,0.275272,4.32022,15.69436775262286,-,34.39720129966736,1.6792171001434326,9.265860080718994,2.7919762134552,281.2519257068634,285.43706798553467,5.0
+google rembert,huggingface tf,TFRemBertModel,0,False,False,False,False,False,False,-,-,6f64f082,-,-,keras,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+google tapas base,huggingface tf,TFTapasModel,0,True,True,True,True,True,True,109360155,4,96b51567,-,-,keras,240687,1.06972,0.880924,0.8235089556145534,1.0979973333333333,0.912942,0.8314610357280772,-,63.77547025680542,5.667855501174927,5.135574579238892,12.050645351409912,639.7542085647583,636.8347005844116,11.0
+google tapas base finetuned sqa,huggingface tf,TFTapasModel,0,True,True,True,True,True,True,109360155,4,96b51567,-,-,keras,240687,1.06972,0.865427,0.8090219870620349,1.0979973333333333,0.897568,0.8174591802287317,-,63.505369901657104,5.831971168518066,8.28011441230774,13.406606435775757,667.1913325786591,565.4348893165588,11.0
+google tapas base finetuned tabfact,huggingface tf,TFTapasModel,0,True,True,True,True,True,True,109360155,4,96b51567,-,-,keras,240687,1.06972,0.853839,0.7981892457839435,1.0979973333333333,0.885872,0.8068070596407034,-,61.805747270584106,5.800036668777466,11.259998321533203,10.939805507659912,632.6435437202454,541.0213866233826,11.0
+google tapas base finetuned wikisql supervised,huggingface tf,TFTapasModel,0,True,True,True,True,True,True,109360155,4,96b51567,-,-,keras,240687,1.06972,0.875274,0.8182271996410275,1.0979973333333333,0.907245,0.8262724985367299,-,61.45381689071655,5.838982105255127,6.577167510986328,12.536357879638672,654.0880038738251,565.6935105323792,11.0
+google tapas base finetuned wtq,huggingface tf,TFTapasModel,0,True,True,True,True,True,True,109360155,4,96b51567,-,-,keras,240687,1.06972,0.869577,0.8129015069363946,1.0979973333333333,0.901924,0.8214264029785137,-,65.29498934745789,5.705482482910156,11.391854286193848,16.720551013946533,663.579179763794,627.9356517791748,11.0
+google tapas mini,huggingface tf,TFTapasModel,0,True,True,True,True,True,True,11156507,1,b57f99e6,-,-,keras,56643,0.06293666666666667,0.221903,3.5258143106826965,0.08571000000000001,0.244325,2.850600863376502,-,8.995656967163086,0.7244088649749756,16.03976345062256,1.1988680362701416,115.80716300010681,48.69931197166443,1.0
+google tapas small,huggingface tf,TFTapasModel,0,True,True,True,True,True,True,28735515,1,7269358b,-,-,keras,102340,0.11371111111111111,0.266317,2.3420490521790116,0.13923644444444447,0.293872,2.1105968424614243,-,18.419354915618896,1.5886242389678955,10.395785093307495,2.8808958530426025,173.21639275550842,116.25198698043823,2.0
+google tapas tiny,huggingface tf,TFTapasModel,0,True,True,True,True,True,True,4333083,1,a8d3801b,-,-,keras,27945,0.03105,0.102991,3.316940418679549,0.05244733333333333,0.124579,2.3753161902098614,-,5.644994258880615,0.37271881103515625,4.235031843185425,0.5040287971496582,45.61987924575806,22.34877920150757,0.7431640625
+googlenet,torch hub vision,GoogLeNet,0,True,True,True,True,True,True,6613040,1,6e59c54b,-,-,pytorch,134590,0.14954444444444445,0.482851,3.2288126903930454,0.18217177777777777,0.52474,2.8804681295919723,-,4.740226984024048,0.6036598682403564,3.836413621902466,0.7972831726074219,198.57864809036255,174.60316467285156,3.0
+gpt1,huggingface,OpenAIGPTModel,0,True,True,True,True,True,True,116159923,4,0342a9fe,-,-,pytorch,269621,1.1983155555555556,0.818671,0.6831848223988488,1.2265395555555556,0.851891,0.6945483300081094,-,21.460368871688843,6.293122291564941,6.746707439422607,13.322429180145264,751.9400782585144,604.7614150047302,11.0
+gpt2,huggingface tf,TFGPT2Model,0,True,True,True,True,True,True,123663898,4,012a10a9,-,-,keras,474558,2.1091466666666667,1.05124,0.49841958201105024,2.333968,1.49662,0.6412341557382106,-,65.98545217514038,6.375670671463013,6.13744592666626,12.718679904937744,873.7094824314117,710.5359001159668,16.0
+gpt2,huggingface,GPT2Model,0,True,True,True,True,True,True,123653827,4,af143a10,-,-,pytorch,482813,2.1458355555555557,1.03035,0.4801626095403396,2.3706675555555554,1.49233,0.6294977954638937,-,19.364166736602783,6.428202390670776,6.969403266906738,12.694334745407104,797.158210515976,689.9647107124329,16.0
+gpt2 doublehead,huggingface,GPT2DoubleHeadsModel,0,True,True,True,True,False,False,162252742,4,7befd733,-,-,pytorch,-,-,-,-,-,-,-,error: 'groq.alloc' op scheduleOp failed,23.518503189086914,8.84312129020691,6.595821380615234,16.880334854125977,0,0,0.0
+gpt2 large,huggingface tf,TFGPT2Model,0,False,False,False,False,False,False,-,-,9d211291,-,-,keras,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+gpt2 medium,huggingface tf,TFGPT2Model,0,True,True,True,True,True,False,353617946,8,73979d23,-,-,keras,1029535,9.151422222222223,-,-,-,-,-,-,195.41101264953613,19.288214206695557,9.478993654251099,38.92202973365784,2312.320233821869,0,49.0
+gpt2 xl,huggingface tf,TFGPT2Model,0,False,False,False,False,False,False,-,-,c0a76325,-,-,keras,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+hardnet39ds,torch hub vision,HarDNet,0,True,True,True,True,True,True,3475460,1,47ba431c,-,-,pytorch,118813,0.13201444444444443,0.692959,5.249114995833789,0.16464177777777778,0.734467,4.461000178164581,-,5.766573905944824,0.3522348403930664,2.9589731693267822,0.48464441299438477,237.1320719718933,192.9362370967865,3.0
+hardnet68,torch hub vision,HarDNet,0,True,True,True,True,True,True,17557702,1,9d6d24cf,-,-,pytorch,429168,0.47685333333333335,1.30944,2.746001565820378,0.5094806666666666,1.35182,2.6533293379794594,-,6.807914733886719,1.1292166709899902,3.3368659019470215,2.0184171199798584,471.28943276405334,436.868145942688,8.0
+hardnet68ds,torch hub vision,HarDNet,0,True,True,True,True,True,True,4162968,1,85f34cd3,-,-,pytorch,193224,0.21469333333333335,-,-,0.2473206666666667,-,-,-,16.044568300247192,0.42147254943847656,3.718498945236206,0.5982918739318848,370.0327401161194,361.73519349098206,5.0
+hardnet85,torch hub vision,HarDNet,0,True,True,True,True,True,False,36657186,1,acb062f3,-,-,pytorch,824538,0.9161533333333334,-,-,-,-,-,-,10.566832065582275,1.9238791465759277,5.06771993637085,3.868123769760132,931.9945597648621,0,15.0
+imagegpt,huggingface,ImageGPTModel,0,True,True,True,True,True,False,75871450,2,3b5850cc,-,-,pytorch,524637,1.16586,-,-,-,-,-,-,43.236891746520996,4.080307483673096,5.226999282836914,8.346592664718628,724.4429631233215,0,12.0
+inception v3,torch hub vision,Inception3,0,True,True,True,True,True,True,23802160,1,46db3db5,-,-,pytorch,245665,0.2729611111111111,0.930916,3.410434534834022,0.30558844444444444,0.972989,3.183984923804565,-,9.979228258132935,1.270094633102417,4.321950674057007,2.661332130432129,296.2814176082611,271.3147768974304,5.0
+junnyu roformer chinese base,huggingface tf,TFRoFormerModel,0,True,True,True,True,False,False,123459107,4,f9889ea2,-,-,keras,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,71.40228390693665,6.944008827209473,6.934747934341431,14.469569206237793,0,0,1.0
+junnyu roformer chinese char base,huggingface tf,TFRoFormerModel,0,True,True,True,True,False,False,94275107,2,23409c26,-,-,keras,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,63.44808888435364,5.043843984603882,7.133379936218262,10.50927186012268,0,0,1.0
+junnyu roformer chinese char small,huggingface tf,TFRoFormerModel,0,True,True,True,True,True,False,15299363,1,9f7a7030,-,-,keras,532843,0.5920477777777777,-,-,-,-,-,-,15.12743330001831,1.344139575958252,10.302040815353394,1.7642979621887207,1455.7091898918152,0,18.0
+junnyu roformer chinese small,huggingface tf,TFRoFormerModel,0,True,True,True,True,True,False,29891363,1,732c9733,-,-,keras,589759,0.6552877777777778,-,-,-,-,-,-,18.60452437400818,2.118295192718506,5.721021413803101,3.5411455631256104,1678.4826610088348,0,19.0
+junnyu roformer small discriminator,huggingface tf,TFRoFormerModel,0,True,True,True,True,False,False,13418277,1,b442427e,-,-,keras,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,21.285916328430176,1.0964436531066895,6.393325090408325,1.7789320945739746,0,0,0.0
+junnyu roformer small generator,huggingface tf,TFRoFormerModel,0,True,True,True,True,False,False,4537953,1,a0b733fa,-,-,keras,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,13.602498531341553,0.41721677780151367,5.243390083312988,0.5897037982940674,0,0,0.0
+keypointrcnn resnet50 fpn,torchvision,KeypointRCNN,0,True,True,False,False,False,False,-,-,2f5908b4,-,-,pytorch,-,-,-,-,-,-,-,-,35.364628076553345,3.7982590198516846,0,0,0,0,0.0
+layoutlm,huggingface,LayoutLMModel,0,True,True,True,True,True,True,112312438,4,33ec397d,-,-,pytorch,341301,1.5168933333333334,0.865695,0.5707026202677402,1.5452666666666668,0.90381,0.5848893394883299,-,12.519816637039185,6.515346527099609,5.988649845123291,12.757697820663452,643.8961026668549,583.7084865570068,13.0
+luke,huggingface,LukeModel,0,True,True,True,True,True,True,124625921,4,431c265c,-,-,pytorch,268448,1.1931022222222223,1.17525,0.9850371394087494,1.2213902222222224,1.21063,0.9911901847366643,-,16.11593198776245,6.289805173873901,6.914344310760498,12.698947668075562,663.8982610702515,594.9758603572845,12.0
+m2m 100,huggingface,M2M100Model,0,True,True,True,True,False,False,484581576,16,533285d2,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,77.07389950752258,31.427856922149658,11.630667209625244,49.82274079322815,0,0,5.0
+marian,huggingface,MarianModel,0,True,True,True,True,True,False,73968235,2,ea99ab2b,-,-,pytorch,869125,1.9313888888888888,-,-,-,-,-,-,15.486350297927856,4.563344717025757,5.369540691375732,8.232971668243408,620.7498936653137,0,13.0
+marianmt,huggingface,MarianMTModel,0,True,True,True,True,True,False,105222373,4,f4dcd1cc,-,-,pytorch,740234,3.289928888888889,-,-,-,-,-,-,16.123725175857544,5.103028059005737,5.49910306930542,10.182902574539185,916.2774879932404,0,18.0
+maskrcnn resnet50 fpn,torchvision,MaskRCNN,0,True,True,False,False,False,False,-,-,a5f78569,-,-,pytorch,-,-,-,-,-,-,-,-,33.60984754562378,2.9362974166870117,0,0,0,0,0.0
+maskrcnn resnet50 fpn v2,torchvision,MaskRCNN,0,True,True,False,False,False,False,-,-,f4f1de9a,-,-,pytorch,-,-,-,-,-,-,-,-,31.089219570159912,2.78718638420105,0,0,0,0,0.0
+megatron bert,huggingface,MegatronBertModel,0,True,True,True,True,True,False,333060302,8,2fa53f3f,-,-,pytorch,478966,4.257475555555556,-,-,-,-,-,-,38.11931347846985,19.420498609542847,7.572920560836792,34.83576273918152,2152.8581726551056,0,35.0
+microsoft layoutlm base uncased,huggingface tf,TFLayoutLMModel,0,True,True,True,True,True,True,109163547,4,822fe59c,-,-,keras,240687,1.06972,0.864792,0.8084283737800546,1.0979973333333333,0.896718,0.81668504355809,-,62.98424458503723,6.08195948600769,7.440324068069458,15.292552709579468,640.1334598064423,635.7571218013763,11.0
+microsoft layoutlm large uncased,huggingface tf,TFLayoutLMModel,0,True,True,True,True,True,False,334557211,8,c208267e,-,-,keras,486329,4.322924444444444,-,-,-,-,-,-,197.38072061538696,21.085325956344604,8.51954460144043,34.84777641296387,1959.3959505558014,0,35.0
+microsoft mpnet base,huggingface tf,TFMPNetModel,0,True,True,True,True,True,True,109562140,4,dcb2e12a,-,-,keras,324584,1.4425955555555556,1.18839,0.8237859845217262,1.470872888888889,1.22057,0.8298269750025986,-,62.98558282852173,5.587937116622925,6.514976978302002,11.499544620513916,648.0264494419098,583.1765701770782,12.0
+minilmv2,huggingface,BertModel,0,True,True,True,True,True,True,22565822,1,f969d36d,-,-,pytorch,253093,0.28121444444444443,0.390688,1.3892885224008567,0.3095664444444445,0.428652,1.3846849608305234,-,3.3469431400299072,1.2698464393615723,5.649458646774292,2.4947564601898193,405.73229932785034,430.8623149394989,7.0
+mnasnet0 5,torch hub vision,MNASNet,0,True,True,True,True,True,True,2200880,1,913218e2,-,-,pytorch,87795,0.09755,0.406253,4.16456176319836,0.13017733333333334,0.447752,3.4395542490756195,-,3.3132057189941406,0.3140130043029785,3.947448968887329,0.35657739639282227,187.90157961845398,139.91546607017517,2.0
+mnasnet0 75,torch hub vision,MNASNet,0,True,True,True,True,True,True,3144288,1,4a915154,-,-,pytorch,113960,0.12662222222222222,0.418825,3.3076737451737452,0.15924955555555553,0.460193,2.8897600272388697,-,3.328479290008545,0.3223404884338379,4.676559925079346,0.48215675354003906,243.21043038368225,174.80763959884644,3.0
+mnasnet1 0,torch hub vision,MNASNet,0,True,True,True,True,True,True,4350160,1,041e693a,-,-,pytorch,114394,0.12710444444444444,0.431254,3.3929104673322033,0.15973177777777775,0.47271,2.9593986029357553,-,3.699366807937622,0.38721776008605957,3.2608797550201416,0.5441029071807861,246.7382698059082,190.67177271842957,3.0
+mnasnet1 3,torch hub vision,MNASNet,0,True,True,True,True,True,True,6239320,1,87ea0deb,-,-,pytorch,126148,0.14016444444444445,0.460924,3.288451659954973,0.17279177777777774,0.502608,2.90874951611638,-,3.9502692222595215,0.5089540481567383,3.4987175464630127,0.7419033050537109,299.08325123786926,231.34418392181396,3.0
+mobilebert,huggingface,MobileBertModel,0,True,True,True,True,True,True,24551994,1,72442a94,-,-,pytorch,226295,0.2514388888888889,2.41012,9.585311208820345,0.2769748888888889,2.43953,8.807765966750296,-,35.51845455169678,1.6705584526062012,5.3931591510772705,2.6606926918029785,288.709823846817,316.8621714115143,5.0
+mobilebert for sequence classification,huggingface,MobileBertForSequenceClassification,0,True,True,True,True,True,True,21062718,1,c6599ac3,-,-,pytorch,223217,0.2480188888888889,2.33626,9.419685776620955,0.2680512222222222,2.35457,8.78403008380239,-,32.4221773147583,1.369438886642456,5.893061637878418,2.338286876678467,272.4625084400177,296.6604073047638,5.0
+mobilenet v2,torch hub vision,MobileNetV2,0,True,True,True,True,True,True,3475078,1,a81033ae,-,-,pytorch,98100,0.109,0.378992,3.4769908256880733,0.14162733333333333,0.420667,2.9702387957126923,-,3.7562310695648193,0.3590357303619385,3.0780766010284424,0.48792147636413574,209.9200189113617,153.09634470939636,2.0
+mobilenet v3 large,torch hub vision,MobileNetV3,0,True,True,True,True,True,True,5457176,1,00777649,-,-,pytorch,100918,0.11213111111111111,0.574761,5.125794209159912,0.14475844444444444,0.616478,4.258666928661233,-,4.166952610015869,0.48605799674987793,4.374546766281128,0.6664433479309082,217.56106233596802,152.6490924358368,2.0
+mobilenet v3 small,torch hub vision,MobileNetV3,0,True,True,True,True,True,True,2529712,1,e7fae853,-,-,pytorch,63265,0.07029444444444445,0.485312,6.903987987038646,0.10292177777777778,0.527029,5.120675248516672,-,2.8044068813323975,0.29749536514282227,5.40753698348999,0.3610410690307617,129.80344414710999,93.34596228599548,2.0
+mobilevit,huggingface,MobileViTModel,0,True,True,True,True,False,False,4913307,1,47b02614,-,-,pytorch,-,-,-,-,-,-,-,error: 'groq.vxm_binary_mask' op scheduleOp failed,7.467036247253418,0.4985980987548828,5.487059593200684,0.6231865882873535,0,0,0.0
+mobilevit small for semantic segmentation,huggingface,MobileViTForSemanticSegmentation,0,True,True,True,True,True,False,6351055,1,5621d1d8,-,-,pytorch,450561,0.5006233333333333,-,-,-,-,-,-,8.47254204750061,0.5985312461853027,6.357140302658081,0.9041388034820557,1168.302723646164,0,14.0
+mobilevit x small for semantic segmentation,huggingface,MobileViTForSemanticSegmentation,0,True,True,True,True,True,True,2938831,1,f9f29c8e,-,-,pytorch,366069,0.40674333333333335,0.971605,2.3887422862902894,0.4396303333333333,1.02308,2.327136965829625,-,7.9012439250946045,0.38953590393066406,5.3212504386901855,0.45520758628845215,800.477157831192,720.3565378189087,10.0
+mobilevit xx small for semantic segmentation,huggingface,MobileViTForSemanticSegmentation,0,True,True,True,True,True,False,1851719,1,535af098,-,-,pytorch,285715,0.31746111111111114,-,-,-,-,-,-,7.812070369720459,0.3241889476776123,5.282257795333862,0.34531402587890625,546.7114744186401,0,8.0
+mpnet,huggingface,MPNetModel,0,True,True,True,True,True,True,109563761,4,747bb620,-,-,pytorch,323721,1.43876,1.15556,0.8031638355250353,1.4670480000000001,1.19078,0.8116844165971392,-,13.005671262741089,5.3586719036102295,6.05635666847229,11.4202561378479,627.902941942215,585.808819770813,12.0
+mt5 base,huggingface,MT5Model,0,True,True,True,True,False,False,393067261,8,6a56180f,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,65.19318747520447,24.097666025161743,7.196679353713989,47.49593114852905,0,0,7.0
+mt5 encoder,huggingface,MT5EncoderModel,0,True,True,True,True,False,False,147030611,4,760f744b,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,12.287190914154053,7.470645189285278,5.436516761779785,16.783090829849243,0,0,4.0
+mt5 small,huggingface,MT5Model,0,True,True,True,True,False,False,173102253,4,9625f18b,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,28.21830987930298,11.163665056228638,6.465817451477051,20.29537534713745,0,0,4.0
+openai doublehead,huggingface,OpenAIGPTDoubleHeadsModel,0,True,True,True,True,True,False,147248566,4,a4df98ec,-,-,pytorch,246247,1.0944311111111111,-,-,-,-,-,-,21.03752827644348,7.4286158084869385,6.140539646148682,15.045311212539673,917.4872500896454,0,14.0
+pegasus,huggingface,PegasusModel,0,True,True,True,True,False,False,403946757,16,b92cca23,-,-,pytorch,-,-,-,-,-,-,-,Groq Compiler exited,65.9071090221405,24.134836196899414,8.800289154052734,44.212358713150024,0,0,0.0
+perceiver,huggingface,PerceiverModel,0,True,True,True,True,True,False,259427302,8,a4732115,-,-,pytorch,515763,4.58456,-,-,-,-,-,-,43.62936329841614,14.00516676902771,6.095951318740845,26.91559386253357,1962.430151939392,0,35.0
+poolformer,huggingface,PoolFormerModel,0,True,True,True,True,False,False,11371452,1,a8cfe755,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,3.2729196548461914,0.7459843158721924,7.49816370010376,6.191686391830444,0,0,0.0
+rag,huggingface,RagModel,0,True,True,True,True,False,False,455991031,16,7e502070,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,71.71414422988892,23.065547943115234,8.668060779571533,46.165016651153564,0,0,4.0
+realm,huggingface,RealmEmbedder,0,True,True,True,True,True,True,109265288,4,d9107239,-,-,pytorch,322635,1.4339333333333333,0.852102,0.5942410153889071,1.4539760000000002,0.870921,0.598992693139364,-,12.05721378326416,5.142676591873169,6.21160101890564,10.45288372039795,635.0441398620605,579.2388248443604,11.0
+regnet x 16gf,torch hub vision,RegNet,0,True,True,True,True,False,False,54171112,2,90fe350f,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,9.061202764511108,2.945756673812866,4.016153573989868,5.488524675369263,0,0,0.0
+regnet x 1 6gf,torch hub vision,RegNet,0,True,True,True,True,False,False,9148224,1,9b6af29e,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,4.554373502731323,0.5817375183105469,4.11007022857666,1.036991834640503,0,0,0.0
+regnet x 32gf,torch hub vision,RegNet,0,True,True,True,True,False,False,107654448,4,024939e4,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,13.221005201339722,4.9432666301727295,4.914834499359131,10.846687078475952,0,0,0.0
+regnet x 3 2gf,torch hub vision,RegNet,0,True,True,True,True,False,False,15235752,1,731da922,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,7.411527633666992,1.0108351707458496,3.7512025833129883,4.0169031620025635,0,0,0.0
+regnet x 400mf,torch hub vision,RegNet,0,True,True,True,True,False,False,5458776,1,08b8712e,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,5.324787378311157,0.43205833435058594,3.666804313659668,0.6097948551177979,0,0,0.0
+regnet x 800mf,torch hub vision,RegNet,0,True,True,True,True,False,False,7223528,1,1e12c62e,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,4.159610271453857,0.540658712387085,3.215935468673706,0.8205082416534424,0,0,0.0
+regnet x 8gf,torch hub vision,RegNet,0,True,True,True,True,False,False,39485176,1,26bfacd7,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,8.139577865600586,1.9938805103302002,3.3433499336242676,9.236977577209473,0,0,0.0
+regnet y 128gf,torch hub vision,RegNet,0,True,True,True,True,False,False,644409734,16,a2a92eba,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,31.599298238754272,2.5166995525360107,8.072551012039185,26.28519558906555,0,0,0.0
+regnet y 16gf,torch hub vision,RegNet,0,True,True,True,True,False,False,83472284,2,a44f744c,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,12.197403907775879,3.945528030395508,3.954571008682251,8.515424966812134,0,0,0.0
+regnet y 1 6gf,torch hub vision,RegNet,0,True,True,True,True,False,False,11151182,1,993181bc,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,13.22936463356018,0.7281465530395508,4.002483606338501,1.3409638404846191,0,0,0.0
+regnet y 32gf,torch hub vision,RegNet,0,True,True,True,True,False,False,144894546,4,16e3920e,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,18.2970974445343,7.5902159214019775,5.288405895233154,16.143948554992676,0,0,0.0
+regnet y 3 2gf,torch hub vision,RegNet,0,True,True,True,True,False,False,19372586,1,a06a50b4,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,10.376834630966187,1.302549123764038,3.4488956928253174,2.0387377738952637,0,0,0.0
+regnet y 400mf,torch hub vision,RegNet,0,True,True,True,True,False,False,4317824,1,74d9ef17,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,6.463138818740845,0.4248197078704834,4.148851633071899,0.5209157466888428,0,0,0.0
+regnet y 800mf,torch hub vision,RegNet,0,True,True,True,True,False,False,6403424,1,efe4b887,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,4.937500715255737,0.4777686595916748,4.514013767242432,0.8035609722137451,0,0,0.0
+regnet y 8gf,torch hub vision,RegNet,0,True,True,True,True,False,False,39298560,1,0c98c39d,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,8.841992139816284,2.054492235183716,4.3792724609375,4.075369596481323,0,0,0.0
+rembert,huggingface,RemBertModel,0,True,True,True,True,True,False,575379982,16,1a69d8de,-,-,pytorch,472922,8.407502222222222,-,-,-,-,-,-,50.0250563621521,3.3596432209014893,8.31409502029419,25.30592942237854,3515.755021572113,0,52.0
+resnet101,torch hub vision,ResNet,0,True,True,True,True,True,True,44447848,1,285cd579,-,-,pytorch,271853,0.3020588888888889,1.1187,3.703582450809813,0.3346862222222222,1.16002,3.4659926909981356,-,13.623885154724121,2.5063092708587646,4.0729498863220215,4.729395389556885,345.8476128578186,379.27320623397827,6.0
+resnet152,torch hub vision,ResNet,0,True,True,True,True,True,True,60045416,2,c732f780,-,-,pytorch,298616,0.6635911111111111,1.60866,2.424173520507943,0.6962184444444445,1.65082,2.371123622439062,-,23.534711599349976,3.0048892498016357,4.391233921051025,6.538663148880005,485.28490805625916,493.15946412086487,8.0
+resnet18,torch hub vision,ResNet,0,True,True,True,True,True,True,11680872,1,11f0e9e3,-,-,pytorch,107465,0.11940555555555556,0.283489,2.3741692644116688,0.15203288888888886,0.324919,2.1371625730105186,-,1.686274766921997,0.6660349369049072,4.877614974975586,1.1731898784637451,136.23655128479004,108.40613746643066,2.0
+resnet34,torch hub vision,ResNet,0,True,True,True,True,True,True,21781608,1,85df0c4a,-,-,pytorch,239580,0.2662,0.512764,1.926235912847483,0.2988273333333334,0.554642,1.8560618060373768,-,3.502830982208252,1.2701702117919922,4.089961290359497,2.2972004413604736,256.85642075538635,243.39231252670288,4.0
+resnet50,torch hub vision,ResNet,0,True,True,True,True,True,True,25507944,1,3ba0a685,-,-,pytorch,189532,0.21059111111111112,0.583106,2.768901293712935,0.24321844444444443,0.624744,2.5686538758482316,-,5.083040237426758,1.3365793228149414,3.857759714126587,2.791001081466675,256.47378063201904,253.05792379379272,4.0
+resnext101 32x8d,torch hub vision,ResNet,0,True,True,True,True,False,False,88592360,2,0b88b3d8,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,16.593698978424072,4.257270336151123,3.7474937438964844,8.751386880874634,0,0,0.0
+resnext50 32x4d,torch hub vision,ResNet,0,True,True,True,True,False,False,24964712,1,ce6f3fb8,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,5.1618030071258545,1.448063850402832,3.402575969696045,2.761382818222046,0,0,0.0
+retinanet resnet50 fpn,torchvision,RetinaNet,0,True,True,False,False,False,False,-,-,7cc11439,-,-,pytorch,-,-,-,-,-,-,-,-,32.117977142333984,2.1181859970092773,0,0,0,0,0.0
+retinanet resnet50 fpn v2,torchvision,RetinaNet,0,True,True,False,False,False,False,-,-,20403119,-,-,pytorch,-,-,-,-,-,-,-,-,28.79207682609558,2.143101930618286,0,0,0,0,0.0
+retribert,huggingface,RetriBertModel,0,True,True,True,True,True,False,81150128,2,4c3ee101,-,-,pytorch,636175,1.4137222222222223,-,-,-,-,-,-,13.008299589157104,4.665558338165283,5.855657339096069,10.12764310836792,804.2204260826111,0,16.0
+roberta base,huggingface tf,TFRobertaModel,0,True,True,True,True,True,True,124622621,4,e21a1cef,-,-,keras,339220,1.5076444444444443,1.16818,0.7748378633335299,1.5359217777777776,1.2003,0.7814851103528421,-,64.62921595573425,6.54569935798645,5.61076545715332,13.305236577987671,669.0087153911591,591.3539757728577,12.0
+roberta,huggingface,RobertaModel,0,True,True,True,True,True,True,109461617,4,f75bf095,-,-,pytorch,269308,1.1969244444444445,1.15155,0.9620908030953407,1.2252124444444443,1.1867,0.9685667211273656,-,13.09390902519226,5.97502875328064,7.380155086517334,12.000783681869507,684.0628018379211,637.4656507968903,11.0
+roberta large,huggingface tf,TFRobertaModel,0,True,True,True,True,True,False,355169309,8,8faddec4,-,-,keras,480534,4.271413333333333,-,-,-,-,-,-,208.93387413024902,19.5819571018219,7.596323728561401,37.230313301086426,2260.563480615616,0,34.0
+roberta large mnli,huggingface tf,TFRobertaModel,0,True,True,True,True,True,False,355169309,8,8faddec4,-,-,keras,480534,4.271413333333333,-,-,-,-,-,-,204.70095419883728,19.30294942855835,7.473975419998169,42.57230854034424,2002.1329522132874,0,34.0
+roformer,huggingface,RoFormerModel,0,True,True,True,True,False,False,123454397,4,a48eefbd,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,15.889728546142578,6.127465486526489,6.15011739730835,12.725799083709717,0,0,1.0
+safety clipvision,stable diffusion,CLIPVisionModel,0,True,True,True,True,True,False,303179946,8,bd5ab0a3,-,-,pytorch,2736828,24.32736,-,-,-,-,-,-,49.145951986312866,17.993385553359985,7.2810282707214355,35.78224968910217,3836.631829023361,0,69.0
+segformer,huggingface,SegformerModel,0,True,True,True,True,True,True,3301437,1,28a23805,-,-,pytorch,264533,0.29392555555555555,1.99644,6.792332147595952,0.32751488888888886,2.03818,6.223167462446153,-,4.895845651626587,0.4581015110015869,4.840421199798584,0.4862213134765625,441.76371598243713,411.9533336162567,7.0
+shufflenet v2 x0 5,torch hub vision,ShuffleNetV2,0,True,True,True,True,True,True,1360111,1,15046a84,-,-,pytorch,135667,0.1507411111111111,0.455888,3.024310996778878,0.18336844444444445,0.497593,2.7136239362643275,-,6.341096878051758,0.25338268280029297,4.718180894851685,0.34278035163879395,246.16475796699524,173.8288357257843,3.0
+shufflenet v2 x1 0,torch hub vision,ShuffleNetV2,0,True,True,True,True,True,True,2263957,1,81185b92,-,-,pytorch,327797,0.36421888888888887,0.607183,1.6670826761684825,0.39684622222222227,0.648962,1.6352984195389426,-,5.7978856563568115,0.2910881042480469,4.164953947067261,0.33498239517211914,777.5801892280579,350.50113582611084,8.0
+shufflenet v2 x1 5,torch hub vision,ShuffleNetV2,0,True,True,True,True,True,True,3481927,1,51805568,-,-,pytorch,563006,0.6255622222222222,0.589121,0.9417464467518996,0.6581895555555556,0.630682,0.9582072439111595,-,5.952707052230835,0.3632345199584961,3.820812702178955,0.5024371147155762,1999.4458410739899,685.7522563934326,13.0
+shufflenet v2 x2 0,torch hub vision,ShuffleNetV2,0,True,True,True,True,True,True,7363285,1,670c36ac,-,-,pytorch,796655,0.8851722222222222,0.684898,0.7737454732600686,0.9177995555555555,0.726832,0.7919289082244538,-,6.3854899406433105,0.5460021495819092,3.7117042541503906,0.8967795372009277,3511.674422264099,1011.8627383708954,16.0
+speech encoder decoder,huggingface,SpeechEncoderDecoderModel,0,False,False,False,False,False,False,-,-,b4dae377,-,-,pytorch,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+speech to text,huggingface,Speech2TextModel,0,True,True,True,True,True,True,29737631,1,fc9ef5d8,-,-,pytorch,221390,0.2459888888888889,1.60262,6.515009711369077,0.3112368888888889,1.76974,5.6861511703125736,-,17.7344970703125,1.7492427825927734,6.088392972946167,3.3483827114105225,199.51442551612854,149.9687066078186,3.0
+splinter,huggingface,SplinterModel,0,True,True,True,True,True,True,108576877,4,d8703a6e,-,-,pytorch,322716,1.4342933333333334,0.836663,0.5833276782062247,1.4625173333333334,0.870087,0.594924231097432,-,12.65497899055481,5.776668310165405,6.0291948318481445,15.412832021713257,713.5170171260834,592.3358278274536,12.0
+squeezebert,huggingface,SqueezeBertModel,0,True,True,True,True,False,False,50775662,2,c54b2d76,-,-,pytorch,-,-,-,-,-,-,-,error: Domain selection failure: Unsupported configuration for convolution,11.323315382003784,2.8925535678863525,7.268703937530518,6.124990701675415,0,0,0.0
+squeezenet1 0,torch hub vision,SqueezeNet,0,True,True,True,True,True,True,1246280,1,8b319b5b,-,-,pytorch,98811,0.10979,0.234215,2.1332999362419165,0.14241733333333334,0.275991,1.937903157855317,-,1.0792701244354248,0.24201321601867676,3.5535972118377686,0.22745776176452637,157.85909867286682,120.09376430511475,2.0
+squeezenet1 1,torch hub vision,SqueezeNet,0,True,True,True,True,True,True,1233288,1,db09563d,-,-,pytorch,67477,0.07497444444444444,0.202208,2.69702565318553,0.10760177777777777,0.244388,2.2712264150943398,-,1.0394976139068604,0.25308775901794434,4.087836027145386,0.22855710983276367,108.97757530212402,68.4293520450592,1.0
+ssd300 vgg16,torchvision,SSDFeatureExtractorVGG,0,True,True,True,True,False,False,22941893,1,ba239042,-,-,pytorch,-,-,-,-,-,-,-,[error]	DecomposeONNXPass failed,2.2249817848205566,1.837364673614502,4.622914552688599,2.231945514678955,0,0,0.3837890625
+ssdlite320 mobilenet v3 large,torchvision,SSDLiteFeatureExtractorMobileNet,0,True,True,True,True,True,True,3531168,1,0b96e723,-,-,pytorch,219343,0.24371444444444446,0.677195,2.778641214900863,0.32106911111111114,0.810732,2.5251012070090826,-,5.28786563873291,0.3817305564880371,3.876810073852539,0.4580545425415039,564.2978217601776,394.2922809123993,6.0
+ssdlite320 mobilenet v3 large,torchvision,SSD,0,True,True,False,True,True,True,3531168,-,cb077411,-,-,pytorch,-,-,-,-,-,-,-,-,69.15801048278809,1.3120176792144775,3.876810073852539,0.4580545425415039,564.2978217601776,394.2922809123993,6.0
+swin b,torch hub vision,SwinTransformer,0,True,True,True,True,True,False,88739576,2,a45575b3,-,-,pytorch,780764,1.7350311111111112,-,-,-,-,-,-,146.30780935287476,6.563915014266968,5.738231420516968,9.599592208862305,2211.9016737937927,0,36.0
+swin s,torch hub vision,SwinTransformer,0,True,True,True,True,True,False,50404826,2,18fbff64,-,-,pytorch,619397,1.3764377777777779,-,-,-,-,-,-,135.89604234695435,4.223651647567749,6.208518028259277,5.649095296859741,1708.1297874450684,0,28.0
+swin t,torch hub vision,SwinTransformer,0,True,True,True,True,True,False,28766996,1,017943b8,-,-,pytorch,602488,0.6694311111111111,-,-,-,-,-,-,36.07523703575134,2.1203391551971436,5.978745937347412,3.192723035812378,1201.6647100448608,0,17.0
+t5 base,huggingface,T5ForConditionalGeneration,0,True,True,True,True,True,False,250329734,8,ba7c8360,-,-,pytorch,1802290,16.020355555555554,-,-,-,-,-,-,51.665544271469116,15.412164449691772,6.9855992794036865,27.52207851409912,2037.0247128009796,0,51.0
+t5 encoder,huggingface,T5EncoderModel,0,True,True,True,True,True,True,35455523,1,0559914f,-,-,pytorch,121926,0.13547333333333333,0.351395,2.5938315043551006,0.160956,0.376338,2.338142100946843,-,5.215369939804077,1.9490394592285156,5.429971218109131,3.9632790088653564,206.2755582332611,158.38948893547058,3.0
+t5 large,huggingface,T5ForConditionalGeneration,0,True,True,True,True,False,False,777382142,16,47d226ef,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,138.3062834739685,4.64010214805603,8.546814203262329,30.870991468429565,0,0,5.0
+t5 small,huggingface,T5ForConditionalGeneration,0,True,True,True,True,True,False,78004298,2,6f1dd5bb,-,-,pytorch,829723,1.843828888888889,-,-,-,-,-,-,16.263088703155518,4.585385322570801,4.356567144393921,10.40729308128357,642.1496911048889,0,12.0
+transfo xl wt103,huggingface tf,TFTransfoXLModel,0,True,True,False,False,False,False,-,-,41a7d660,-,-,keras,-,-,-,-,-,-,-,-,188.56737732887268,16.702563285827637,0,0,0,0,0.0
+turkunlp bert base finnish cased v1,huggingface tf,TFBertModel,0,True,True,True,True,True,True,124203291,4,ff592629,-,-,keras,255013,1.133391111111111,0.879039,0.7755831075278516,1.1616684444444443,0.911155,0.7843503061114399,-,67.88579392433167,6.356027841567993,11.41984224319458,14.169193983078003,698.3321399688721,593.042051076889,12.0
+turkunlp bert base finnish uncased v1,huggingface tf,TFBertModel,0,True,True,True,True,True,True,124200219,4,a433a9be,-,-,keras,255013,1.133391111111111,0.853199,0.7527842698215386,1.1616684444444443,0.885749,0.762480038289755,-,74.60403728485107,7.848267555236816,6.763267278671265,15.650741338729858,686.8622989654541,595.8037488460541,12.0
+unet 2d condition,stable diffusion,UNet2DConditionModel,0,True,True,True,True,False,False,859526310,32,8d97aa42,-,-,pytorch,-,-,-,-,-,-,-,[error]	UserMessagingPass failed,133.1716365814209,5.124599456787109,13.050671100616455,42.99677896499634,0,0,3.0
+unet,torch hub,UNet,0,True,True,True,True,False,False,7760097,1,a76ab7f4,-,-,pytorch,-,-,-,-,-,-,-,error: 'groq.alloc' op scheduleOp failed,1.921447515487671,0.5223731994628906,4.001353979110718,0.8359994888305664,0,0,0.0
+vae decoder,stable diffusion,Decoder,0,True,True,True,True,False,False,49492344,1,d2afe38b,-,-,pytorch,-,-,-,-,-,-,-,[error]	UserMessagingPass failed,9.975871324539185,2.445509433746338,11.146234035491943,4.812315225601196,0,0,0.4052734375
+vgg11,torch hub vision,VGG,0,True,True,True,True,False,False,132857448,4,b38617af,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,8.866010904312134,6.827158212661743,4.598454475402832,13.742358684539795,0,0,3.0
+vgg11 bn,torch hub vision,VGG,0,True,True,True,True,False,False,132857448,4,08550040,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,8.419323444366455,11.85395336151123,6.216663122177124,14.475058317184448,0,0,3.0
+vgg13,torch hub vision,VGG,0,True,True,True,True,False,False,133041768,4,20ce33fd,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,7.601443290710449,6.465792894363403,4.460560321807861,14.70713758468628,0,0,3.0
+vgg13 bn,torch hub vision,VGG,0,True,True,True,True,False,False,133041768,4,20dffe7e,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,8.82175874710083,7.084183931350708,4.1585774421691895,14.24393105506897,0,0,3.0
+vgg16,torch hub vision,VGG,0,True,True,True,True,False,False,138350184,4,b628f277,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,8.594246625900269,7.093656778335571,4.520263433456421,15.226082563400269,0,0,3.0
+vgg16 bn,torch hub vision,VGG,0,True,True,True,True,False,False,138350184,4,8e2b426b,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,9.67589020729065,7.274411678314209,4.67074728012085,15.221680641174316,0,0,3.0
+vgg19 bn,torch hub vision,VGG,0,True,True,True,True,False,False,143658600,4,bc2392e4,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,9.57696795463562,12.258500099182129,4.258410692214966,20.991580486297607,0,0,3.0
+vgg19,torch hub vision,VGG,0,True,True,True,True,False,False,143658600,4,d889f054,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,10.145533800125122,7.39092493057251,8.056560754776001,15.576763391494751,0,0,3.0
+vit,huggingface,ViTModel,0,True,True,True,True,True,False,86271085,2,993623dd,-,-,pytorch,312967,0.6954822222222222,-,-,-,-,-,-,10.372016906738281,4.317581415176392,5.847078323364258,9.953554391860962,798.6206314563751,0,15.0
+vit b 16,torch hub vision,VisionTransformer,0,True,True,True,True,True,False,86497170,2,03284b2a,-,-,pytorch,347281,0.7717355555555555,-,-,-,-,-,-,16.359175205230713,4.469055652618408,6.489959001541138,8.717131614685059,798.6858403682709,0,15.0
+vit b 32,torch hub vision,VisionTransformer,0,True,True,True,True,True,True,88153746,2,0f269397,-,-,pytorch,141379,0.31417555555555554,1.21704,3.873757771663401,0.3468028888888889,1.25706,3.624710290123176,-,11.437183618545532,4.531003713607788,6.779010534286499,9.370833158493042,372.45084500312805,245.83010864257812,5.0
+vit h 14,torch hub vision,VisionTransformer,0,True,True,True,True,True,False,631723670,16,eb5a7cc3,-,-,pytorch,908734,16.155271111111112,-,-,-,-,-,-,49.59873175621033,3.659210681915283,7.889199733734131,27.104483366012573,5166.438497543335,0,92.0
+vit l 16,torch hub vision,VisionTransformer,0,True,True,True,True,True,False,304134446,8,8426a685,-,-,pytorch,668168,5.939271111111111,-,-,-,-,-,-,37.940929889678955,14.659139633178711,6.787899017333984,28.3262996673584,2512.194174051285,0,47.0
+vit l 32,torch hub vision,VisionTransformer,0,True,True,True,True,True,True,306343214,8,2ff53b19,-,-,pytorch,298412,2.6525511111111113,2.36527,0.891696295725373,2.685178444444445,2.40535,0.895787765977564,-,34.9265673160553,14.679825067520142,7.856443166732788,32.06491756439209,1146.756999015808,687.7367420196533,16.0
+wide resnet101 2,torch hub vision,ResNet,0,True,True,True,True,True,False,126752872,4,0eb07645,-,-,pytorch,602936,2.6797155555555556,-,-,-,-,-,-,18.75733184814453,6.966927766799927,4.561936378479004,13.583825588226318,828.5841436386108,0,15.0
+wide resnet50 2,torch hub vision,ResNet,0,True,True,True,True,True,True,68819048,2,fd743f94,-,-,pytorch,312588,0.69464,0.758363,1.0917352873430841,0.7272673333333333,0.800511,1.1007107886050156,-,13.268449783325195,3.3691229820251465,4.80395245552063,7.447877645492554,496.3272604942322,514.1218695640564,7.0
+wietsedv bert base dutch cased,huggingface tf,TFBertModel,0,True,True,True,True,True,True,108818715,4,a10974d8,-,-,keras,240137,1.0672755555555555,0.865948,0.8113630968988537,1.0955528888888888,0.897974,0.8196537192382619,-,66.6110405921936,6.046767473220825,5.977190732955933,11.518624067306519,684.1529595851898,547.5567286014557,11.0
+xglm,huggingface,XGLMModel,0,True,True,True,True,False,False,566264069,16,41f01198,-,-,pytorch,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,38.38829159736633,2.821486473083496,9.108213901519775,24.847039222717285,0,0,9.0
+xlm,huggingface,XLMModel,0,True,True,True,True,False,False,665991362,16,6918ed2c,-,-,pytorch,-,-,-,-,-,-,-,TERMINATING WITH SIGNAL: 6,30.61037802696228,3.484970808029175,9.273037433624268,29.04970622062683,0,0,0.0
+xlm clm ende 1024,huggingface tf,TFXLMModel,0,True,True,True,True,True,True,141882524,4,6c4dd7ed,-,-,keras,224856,0.99936,0.525332,0.5256684277937881,1.0303039999999999,0.560667,0.5441762819517347,-,64.17615556716919,7.34126877784729,6.079793691635132,14.386667013168335,545.8984005451202,435.23331475257874,10.0
+xlm clm enfr 1024,huggingface tf,TFXLMModel,0,True,True,True,True,True,True,141309084,4,7fc75f6a,-,-,keras,224028,0.99568,0.5065,0.5086975735175959,1.026624,0.541606,0.5275602362695593,-,68.33922410011292,7.9643638134002686,11.622196674346924,15.443050622940063,551.6315245628357,447.6004943847656,9.0
+xlm mlm 100 1280,huggingface tf,TFXLMModel,0,False,False,False,False,False,False,-,-,37e39268,-,-,keras,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+xlm mlm 17 1280,huggingface tf,TFXLMModel,0,False,False,False,False,False,False,-,-,37e39268,-,-,keras,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+xlm mlm en 2048,huggingface tf,TFXLMModel,0,False,False,False,False,False,False,-,-,0b3c22a3,-,-,keras,-,-,-,-,-,-,-,-,0,0,0,0,0,0,0.0
+xlm mlm ende 1024,huggingface tf,TFXLMModel,0,True,True,True,True,True,True,141882524,4,6c4dd7ed,-,-,keras,224856,0.99936,0.509037,0.5093629923150816,1.0303039999999999,0.544147,0.5281421793955959,-,67.11615180969238,7.436736106872559,6.558057546615601,15.560957908630371,585.748512506485,458.64964413642883,10.0
+xlm mlm enfr 1024,huggingface tf,TFXLMModel,0,True,True,True,True,True,True,141309084,4,7fc75f6a,-,-,keras,224028,0.99568,0.510332,0.512546199582195,1.026624,0.545592,0.5314428651580325,-,68.03069424629211,7.565172433853149,10.35197114944458,16.609147310256958,607.6382949352264,483.14567828178406,9.0
+xlm mlm enro 1024,huggingface tf,TFXLMModel,0,True,True,True,True,True,True,141772956,4,790bc5d8,-,-,keras,224281,0.9968044444444445,0.512824,0.5144680111110614,1.0277484444444442,0.547889,0.533096404048721,-,62.57867431640625,6.761414051055908,6.3866307735443115,14.829219818115234,565.837028503418,419.2949640750885,10.0
+xlm mlm tlm xnli15 1024,huggingface tf,TFXLMModel,0,True,True,True,True,False,False,248408220,8,7845125b,-,-,keras,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,122.6650013923645,13.568789958953857,9.39238977432251,27.095964670181274,0,0,4.0
+xlm mlm xnli15 1024,huggingface tf,TFXLMModel,0,True,True,True,True,False,False,248408220,8,7845125b,-,-,keras,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,127.58242225646973,13.051843166351318,13.837263345718384,25.420574188232422,0,0,4.0
+xlm roberta,huggingface,XLMRobertaModel,0,True,True,True,True,True,True,109461617,4,a0532c05,-,-,pytorch,269308,1.1969244444444445,1.15669,0.9663851426619335,1.2252124444444443,1.19215,0.973014929292988,-,11.942494630813599,5.38583517074585,6.649282932281494,11.078773975372314,629.4306483268738,593.8053004741669,11.0
+xlm roberta base,huggingface tf,TFXLMRobertaModel,0,True,True,True,True,False,False,278020637,8,c26fa3d0,-,-,keras,-,-,-,-,-,-,-,[error]	PhaseAllocationPass failed,101.63868832588196,14.859676361083984,6.532133340835571,30.734724283218384,0,0,7.0
+xlnet,huggingface,XLNetModel,0,True,True,True,True,False,False,341121821,8,5cfcb429,-,-,pytorch,-,-,-,-,-,-,-,[error]	DecomposeONNXToONNXPass failed,46.60349106788635,16.23669171333313,7.605469465255737,33.831074237823486,0,0,1.0
+xlnet base cased,huggingface tf,TFXLNetModel,0,True,True,True,True,False,False,111871148,4,d8ee3819,-,-,keras,-,-,-,-,-,-,-,[error]	DecomposeONNXToONNXPass failed,87.03524327278137,7.773357629776001,7.191736698150635,13.637510776519775,0,0,1.0
+xlnet large cased,huggingface tf,TFXLNetModel,0,True,True,True,True,False,False,341051331,8,8d4015a4,-,-,keras,-,-,-,-,-,-,-,[error]	DecomposeONNXToONNXPass failed,253.47610688209534,17.301578521728516,9.008344173431396,34.86139726638794,0,0,1.0
+yitutech conv bert base,huggingface tf,TFConvBertModel,0,True,True,True,True,False,False,105366619,4,6bbc4b16,-,-,keras,-,-,-,-,-,-,-,Groq Compiler exited,67.05587720870972,5.753269195556641,11.61150598526001,14.101041555404663,0,0,0.0
+yitutech conv bert medium small,huggingface tf,TFConvBertModel,0,True,True,True,True,False,False,17391588,1,20363de2,-,-,keras,-,-,-,-,-,-,-,Groq Compiler exited,25.395699501037598,1.2469260692596436,10.5399169921875,2.2343053817749023,0,0,0.0
+yitutech conv bert small,huggingface tf,TFConvBertModel,0,True,True,True,True,False,False,13055835,1,7a0200b5,-,-,keras,-,-,-,-,-,-,-,Groq Compiler exited,21.26344132423401,0.9432640075683594,11.383577108383179,5.448837995529175,0,0,0.0
+yolos tiny for object detection,huggingface,YolosForObjectDetection,0,True,True,True,True,False,False,6489028,1,8f6a6a55,-,-,pytorch,-,-,-,-,-,-,-,[error]	groq::ONNXToGroqNNPass failed,5.707167387008667,0.6715781688690186,5.233812570571899,0.7539584636688232,0,0,0.1826171875