Refactor

app.py

@@ -8,8 +8,7 @@ from datasets import get_dataset_config_names
 from dotenv import load_dotenv
 from huggingface_hub import list_datasets
 
-from evaluation import (EvaluationInfo, compute_evaluation_id,
-                        get_evaluation_ids)
+from evaluation import EvaluationInfo, filter_evaluated_models
 from utils import (get_compatible_models, get_key, get_metadata, http_get,
                    http_post)
 
@@ -247,82 +246,73 @@ with st.form(key="form"):
     selected_models = st.multiselect("Select the models you wish to evaluate", compatible_models)
     print("Selected models:", selected_models)
 
-    evaluation_ids = get_evaluation_ids()
-
-    for idx, model in enumerate(selected_models):
-        eval_info = EvaluationInfo(
-            task=selected_task,
-            model=model,
-            dataset_name=selected_dataset,
-            dataset_config=selected_config,
-            dataset_split=selected_split,
-        )
-        candidate_id = hash(eval_info)
-        if candidate_id in evaluation_ids:
-            st.info(f"Model {model} has already been evaluated on this configuration. Skipping ...")
-            selected_models.pop(idx)
-
+    selected_models = filter_evaluated_models(
+        selected_models, selected_task, selected_dataset, selected_config, selected_split
+    )
     print("Selected models:", selected_models)
 
     submit_button = st.form_submit_button("Make submission")
 
     if submit_button:
-        project_id = str(uuid.uuid4())[:3]
-        payload = {
-            "username": AUTOTRAIN_USERNAME,
-            "proj_name": f"my-eval-project-{project_id}",
-            "task": TASK_TO_ID[selected_task],
-            "config": {
-                "language": "en",
-                "max_models": 5,
-                "instance": {
-                    "provider": "aws",
-                    "instance_type": "ml.g4dn.4xlarge",
-                    "max_runtime_seconds": 172800,
-                    "num_instances": 1,
-                    "disk_size_gb": 150,
-                },
-                "evaluation": {
-                    "metrics": [],
-                    "models": selected_models,
-                },
-            },
-        }
-        print(f"Payload: {payload}")
-        project_json_resp = http_post(
-            path="/projects/create", payload=payload, token=HF_TOKEN, domain=AUTOTRAIN_BACKEND_API
-        ).json()
-        print(project_json_resp)
-
-        if project_json_resp["created"]:
+        if len(selected_models) > 0:
+            project_id = str(uuid.uuid4())[:3]
             payload = {
-                "split": 4,  # use "auto" split choice in AutoTrain
-                "col_mapping": col_mapping,
-                "load_config": {"max_size_bytes": 0, "shuffle": False},
+                "username": AUTOTRAIN_USERNAME,
+                "proj_name": f"my-eval-project-{project_id}",
+                "task": TASK_TO_ID[selected_task],
+                "config": {
+                    "language": "en",
+                    "max_models": 5,
+                    "instance": {
+                        "provider": "aws",
+                        "instance_type": "ml.g4dn.4xlarge",
+                        "max_runtime_seconds": 172800,
+                        "num_instances": 1,
+                        "disk_size_gb": 150,
+                    },
+                    "evaluation": {
+                        "metrics": [],
+                        "models": selected_models,
+                    },
+                },
             }
-            data_json_resp = http_post(
-                path=f"/projects/{project_json_resp['id']}/data/{selected_dataset}",
-                payload=payload,
-                token=HF_TOKEN,
-                domain=AUTOTRAIN_BACKEND_API,
-                params={"type": "dataset", "config_name": selected_config, "split_name": selected_split},
+            print(f"Payload: {payload}")
+            project_json_resp = http_post(
+                path="/projects/create", payload=payload, token=HF_TOKEN, domain=AUTOTRAIN_BACKEND_API
             ).json()
-            print(data_json_resp)
-            if data_json_resp["download_status"] == 1:
-                train_json_resp = http_get(
-                    path=f"/projects/{project_json_resp['id']}/data/start_process",
+            print(project_json_resp)
+
+            if project_json_resp["created"]:
+                payload = {
+                    "split": 4,  # use "auto" split choice in AutoTrain
+                    "col_mapping": col_mapping,
+                    "load_config": {"max_size_bytes": 0, "shuffle": False},
+                }
+                data_json_resp = http_post(
+                    path=f"/projects/{project_json_resp['id']}/data/{selected_dataset}",
+                    payload=payload,
                     token=HF_TOKEN,
                     domain=AUTOTRAIN_BACKEND_API,
+                    params={"type": "dataset", "config_name": selected_config, "split_name": selected_split},
                 ).json()
-                print(train_json_resp)
-                if train_json_resp["success"]:
-                    st.success(f"✅ Successfully submitted evaluation job with project ID {project_id}")
-                    st.markdown(
-                        f"""
-                    Evaluation takes appoximately 1 hour to complete, so grab a ☕ or 🍵 while you wait:
-
-                    * 📊 Click [here](https://huggingface.co/spaces/autoevaluate/leaderboards) to view the results from your submission
-                    """
-                    )
-                else:
-                    st.error("🙈 Oh noes, there was an error submitting your submission!")
+                print(data_json_resp)
+                if data_json_resp["download_status"] == 1:
+                    train_json_resp = http_get(
+                        path=f"/projects/{project_json_resp['id']}/data/start_process",
+                        token=HF_TOKEN,
+                        domain=AUTOTRAIN_BACKEND_API,
+                    ).json()
+                    print(train_json_resp)
+                    if train_json_resp["success"]:
+                        st.success(f"✅ Successfully submitted evaluation job with project ID {project_id}")
+                        st.markdown(
+                            f"""
+                        Evaluation takes appoximately 1 hour to complete, so grab a ☕ or 🍵 while you wait:
+
+                        * 📊 Click [here](https://huggingface.co/spaces/autoevaluate/leaderboards) to view the results from your submission
+                        """
+                        )
+                    else:
+                        st.error("🙈 Oh noes, there was an error submitting your evaluation job!")
+        else:
+            st.warning("⚠️ No models were selected for evaluation!")

evaluation.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 
+import streamlit as st
 from huggingface_hub import DatasetFilter, HfApi
 from huggingface_hub.hf_api import DatasetInfo
 
@@ -24,3 +25,22 @@ def get_evaluation_ids():
     filt = DatasetFilter(author="autoevaluate")
     evaluation_datasets = HfApi().list_datasets(filter=filt, full=True)
     return [compute_evaluation_id(dset) for dset in evaluation_datasets]
+
+
+def filter_evaluated_models(models, task, dataset_name, dataset_config, dataset_split):
+    evaluation_ids = get_evaluation_ids()
+
+    for idx, model in enumerate(models):
+        evaluation_info = EvaluationInfo(
+            task=task,
+            model=model,
+            dataset_name=dataset_name,
+            dataset_config=dataset_config,
+            dataset_split=dataset_split,
+        )
+        candidate_id = hash(evaluation_info)
+        if candidate_id in evaluation_ids:
+            st.info(f"Model {model} has already been evaluated on this configuration. Skipping evaluation...")
+            models.pop(idx)
+
+    return models