Fix data loading
Browse files
data.py
CHANGED
|
@@ -34,6 +34,20 @@ IMPORTANT_MODELS = [
|
|
| 34 |
"qwen2_5_omni", # new (omnimodal models)
|
| 35 |
]
|
| 36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
def read_one_dataframe(json_path: str, device_label: str) -> pd.DataFrame:
|
| 39 |
df = pd.read_json(json_path, orient="index")
|
|
@@ -56,20 +70,7 @@ def get_distant_data() -> pd.DataFrame:
|
|
| 56 |
df_nvidia = read_one_dataframe(nvidia_path, "nvidia")
|
| 57 |
# Join both dataframes
|
| 58 |
joined = df_amd.join(df_nvidia, rsuffix="_nvidia", lsuffix="_amd", how="outer")
|
| 59 |
-
joined = joined[
|
| 60 |
-
[
|
| 61 |
-
"success_amd",
|
| 62 |
-
"success_nvidia",
|
| 63 |
-
"failed_multi_no_amd",
|
| 64 |
-
"failed_multi_no_nvidia",
|
| 65 |
-
"failed_single_no_amd",
|
| 66 |
-
"failed_single_no_nvidia",
|
| 67 |
-
"failures_amd",
|
| 68 |
-
"failures_nvidia",
|
| 69 |
-
"job_link_amd",
|
| 70 |
-
"job_link_nvidia",
|
| 71 |
-
]
|
| 72 |
-
]
|
| 73 |
joined.index = joined.index.str.replace("^models_", "", regex=True)
|
| 74 |
# Fitler out all but important models
|
| 75 |
important_models_lower = [model.lower() for model in IMPORTANT_MODELS]
|
|
@@ -83,22 +84,7 @@ def get_sample_data() -> pd.DataFrame:
|
|
| 83 |
df_nvidia = read_one_dataframe("sample_nvidia.json", "nvidia")
|
| 84 |
# Join both dataframes
|
| 85 |
joined = df_amd.join(df_nvidia, rsuffix="_nvidia", lsuffix="_amd", how="outer")
|
| 86 |
-
joined = joined[
|
| 87 |
-
[
|
| 88 |
-
"success_amd",
|
| 89 |
-
"success_nvidia",
|
| 90 |
-
"skipped_amd",
|
| 91 |
-
"skipped_nvidia",
|
| 92 |
-
"failed_multi_no_amd",
|
| 93 |
-
"failed_multi_no_nvidia",
|
| 94 |
-
"failed_single_no_amd",
|
| 95 |
-
"failed_single_no_nvidia",
|
| 96 |
-
"failures_amd",
|
| 97 |
-
"failures_nvidia",
|
| 98 |
-
"job_link_amd",
|
| 99 |
-
"job_link_nvidia",
|
| 100 |
-
]
|
| 101 |
-
]
|
| 102 |
joined.index = joined.index.str.replace("^models_", "", regex=True)
|
| 103 |
# Fitler out all but important models
|
| 104 |
important_models_lower = [model.lower() for model in IMPORTANT_MODELS]
|
|
|
|
| 34 |
"qwen2_5_omni", # new (omnimodal models)
|
| 35 |
]
|
| 36 |
|
| 37 |
+
KEYS_TO_KEEP = [
|
| 38 |
+
"success_amd",
|
| 39 |
+
"success_nvidia",
|
| 40 |
+
"skipped_amd",
|
| 41 |
+
"skipped_nvidia",
|
| 42 |
+
"failed_multi_no_amd",
|
| 43 |
+
"failed_multi_no_nvidia",
|
| 44 |
+
"failed_single_no_amd",
|
| 45 |
+
"failed_single_no_nvidia",
|
| 46 |
+
"failures_amd",
|
| 47 |
+
"failures_nvidia",
|
| 48 |
+
"job_link_amd",
|
| 49 |
+
"job_link_nvidia",
|
| 50 |
+
]
|
| 51 |
|
| 52 |
def read_one_dataframe(json_path: str, device_label: str) -> pd.DataFrame:
|
| 53 |
df = pd.read_json(json_path, orient="index")
|
|
|
|
| 70 |
df_nvidia = read_one_dataframe(nvidia_path, "nvidia")
|
| 71 |
# Join both dataframes
|
| 72 |
joined = df_amd.join(df_nvidia, rsuffix="_nvidia", lsuffix="_amd", how="outer")
|
| 73 |
+
joined = joined[KEYS_TO_KEEP]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
joined.index = joined.index.str.replace("^models_", "", regex=True)
|
| 75 |
# Fitler out all but important models
|
| 76 |
important_models_lower = [model.lower() for model in IMPORTANT_MODELS]
|
|
|
|
| 84 |
df_nvidia = read_one_dataframe("sample_nvidia.json", "nvidia")
|
| 85 |
# Join both dataframes
|
| 86 |
joined = df_amd.join(df_nvidia, rsuffix="_nvidia", lsuffix="_amd", how="outer")
|
| 87 |
+
joined = joined[KEYS_TO_KEEP]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
joined.index = joined.index.str.replace("^models_", "", regex=True)
|
| 89 |
# Fitler out all but important models
|
| 90 |
important_models_lower = [model.lower() for model in IMPORTANT_MODELS]
|