autoevaluator
HF staff
commited on
Commit
580b4e4
1 Parent(s): dde0fd4

Add local dev

Browse files
Files changed (5) hide show
  1. .env.example +0 -4
  2. .env.template +4 -0
  3. README.md +6 -0
  4. app.py +67 -41
  5. run_evaluation_jobs.py +24 -13
.env.example DELETED
@@ -1,4 +0,0 @@
1
- AUTOTRAIN_USERNAME=autoevaluator # The bot that authors evaluation jobs
2
- HF_TOKEN=hf_xxx # An API token of the `autoevaluator` user
3
- AUTOTRAIN_BACKEND_API=https://api-staging.autotrain.huggingface.co # The AutoTrain backend to send jobs to. Use https://api.autotrain.huggingface.co for prod
4
- DATASETS_PREVIEW_API=https://datasets-server.huggingface.co # The API to grab dataset information from
 
 
 
 
.env.template ADDED
@@ -0,0 +1,4 @@
 
 
 
 
1
+ AUTOTRAIN_USERNAME=autoevaluator # The bot or user that authors evaluation jobs
2
+ HF_TOKEN=hf_xxx # An API token of the `autoevaluator` user
3
+ AUTOTRAIN_BACKEND_API=https://api-staging.autotrain.huggingface.co # The AutoTrain backend to send jobs to. Use https://api.autotrain.huggingface.co for prod or http://localhost:8000 for local development
4
+ DATASETS_PREVIEW_API=https://datasets-server.huggingface.co # The API to grab dataset information from
README.md CHANGED
@@ -54,4 +54,10 @@ Models are evaluated by AutoTrain, with the payload sent to the `AUTOTRAIN_BACKE
54
 
55
  ```
56
  AUTOTRAIN_BACKEND_API=https://api.autotrain.huggingface.co
 
 
 
 
 
 
57
  ```
54
 
55
  ```
56
  AUTOTRAIN_BACKEND_API=https://api.autotrain.huggingface.co
57
+ ```
58
+
59
+ To evaluate models with a _local_ instance of AutoTrain, change the environment to:
60
+
61
+ ```
62
+ AUTOTRAIN_BACKEND_API=http://localhost:8000
63
  ```
app.py CHANGED
@@ -1,4 +1,5 @@
1
  import os
 
2
  from pathlib import Path
3
 
4
  import pandas as pd
@@ -515,47 +516,72 @@ with st.form(key="form"):
515
  token=HF_TOKEN,
516
  domain=AUTOTRAIN_BACKEND_API,
517
  ).json()
518
- print(f"INFO -- AutoTrain job response: {train_json_resp}")
519
- if train_json_resp["success"]:
520
- train_eval_index = {
521
- "train-eval-index": [
522
- {
523
- "config": selected_config,
524
- "task": AUTOTRAIN_TASK_TO_HUB_TASK[selected_task],
525
- "task_id": selected_task,
526
- "splits": {"eval_split": selected_split},
527
- "col_mapping": col_mapping,
528
- }
529
- ]
530
- }
531
- selected_metadata = yaml.dump(train_eval_index, sort_keys=False)
532
- dataset_card_url = get_dataset_card_url(selected_dataset)
533
- st.success("✅ Successfully submitted evaluation job!")
534
- st.markdown(
535
- f"""
536
- Evaluation can take up to 1 hour to complete, so grab a ☕️ or 🍵 while you wait:
537
-
538
- * 🔔 A [Hub pull request](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) with the evaluation results will be opened for each model you selected. Check your email for notifications.
539
- * 📊 Click [here](https://hf.co/spaces/autoevaluate/leaderboards?dataset={selected_dataset}) to view the results from your submission once the Hub pull request is merged.
540
- * 🥱 Tired of configuring evaluations? Add the following metadata to the [dataset card]({dataset_card_url}) to enable 1-click evaluations:
541
- """ # noqa
542
- )
543
- st.markdown(
544
- f"""
545
- ```yaml
546
- {selected_metadata}
547
- """
548
- )
549
- print("INFO -- Pushing evaluation job logs to the Hub")
550
- evaluation_log = {}
551
- evaluation_log["project_id"] = project_json_resp["id"]
552
- evaluation_log["is_evaluated"] = False
553
- evaluation_log["payload"] = project_payload
554
- evaluation_log["project_creation_response"] = project_json_resp
555
- evaluation_log["dataset_creation_response"] = data_json_resp
556
- evaluation_log["autotrain_job_response"] = train_json_resp
557
- commit_evaluation_log(evaluation_log, hf_access_token=HF_TOKEN)
558
  else:
559
- st.error("🙈 Oh no, there was an error submitting your evaluation job!")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
560
  else:
561
  st.warning("⚠️ No models left to evaluate! Please select other models and try again.")
1
  import os
2
+ import time
3
  from pathlib import Path
4
 
5
  import pandas as pd
516
  token=HF_TOKEN,
517
  domain=AUTOTRAIN_BACKEND_API,
518
  ).json()
519
+ # For local development we process and approve projects on-the-fly
520
+ if "localhost" in AUTOTRAIN_BACKEND_API:
521
+ with st.spinner("⏳ Waiting for data processing to complete ..."):
522
+ is_data_processing_success = False
523
+ while is_data_processing_success is not True:
524
+ project_status = http_get(
525
+ path=f"/projects/{project_json_resp['id']}",
526
+ token=HF_TOKEN,
527
+ domain=AUTOTRAIN_BACKEND_API,
528
+ ).json()
529
+ if project_status["status"] == 3:
530
+ is_data_processing_success = True
531
+ time.sleep(10)
532
+
533
+ # Approve training job
534
+ train_job_resp = http_post(
535
+ path=f"/projects/{project_json_resp['id']}/start_training",
536
+ token=HF_TOKEN,
537
+ domain=AUTOTRAIN_BACKEND_API,
538
+ ).json()
539
+ st.success("✅ Data processing and project approval complete - go forth and evaluate!")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
540
  else:
541
+ # Prod/staging submissions are evaluated in a cron job via the run_evaluation_jobs.py script
542
+ print(f"INFO -- AutoTrain job response: {train_json_resp}")
543
+ if train_json_resp["success"]:
544
+ train_eval_index = {
545
+ "train-eval-index": [
546
+ {
547
+ "config": selected_config,
548
+ "task": AUTOTRAIN_TASK_TO_HUB_TASK[selected_task],
549
+ "task_id": selected_task,
550
+ "splits": {"eval_split": selected_split},
551
+ "col_mapping": col_mapping,
552
+ }
553
+ ]
554
+ }
555
+ selected_metadata = yaml.dump(train_eval_index, sort_keys=False)
556
+ dataset_card_url = get_dataset_card_url(selected_dataset)
557
+ st.success("✅ Successfully submitted evaluation job!")
558
+ st.markdown(
559
+ f"""
560
+ Evaluation can take up to 1 hour to complete, so grab a ☕️ or 🍵 while you wait:
561
+
562
+ * 🔔 A [Hub pull request](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) with the evaluation results will be opened for each model you selected. Check your email for notifications.
563
+ * 📊 Click [here](https://hf.co/spaces/autoevaluate/leaderboards?dataset={selected_dataset}) to view the results from your submission once the Hub pull request is merged.
564
+ * 🥱 Tired of configuring evaluations? Add the following metadata to the [dataset card]({dataset_card_url}) to enable 1-click evaluations:
565
+ """ # noqa
566
+ )
567
+ st.markdown(
568
+ f"""
569
+ ```yaml
570
+ {selected_metadata}
571
+ """
572
+ )
573
+ print("INFO -- Pushing evaluation job logs to the Hub")
574
+ evaluation_log = {}
575
+ evaluation_log["project_id"] = project_json_resp["id"]
576
+ evaluation_log["autotrain_env"] = (
577
+ "staging" if "staging" in AUTOTRAIN_BACKEND_API else "prod"
578
+ )
579
+ evaluation_log["payload"] = project_payload
580
+ evaluation_log["project_creation_response"] = project_json_resp
581
+ evaluation_log["dataset_creation_response"] = data_json_resp
582
+ evaluation_log["autotrain_job_response"] = train_json_resp
583
+ commit_evaluation_log(evaluation_log, hf_access_token=HF_TOKEN)
584
+ else:
585
+ st.error("🙈 Oh no, there was an error submitting your evaluation job!")
586
  else:
587
  st.warning("⚠️ No models left to evaluate! Please select other models and try again.")
run_evaluation_jobs.py CHANGED
@@ -16,28 +16,39 @@ AUTOTRAIN_TOKEN = os.getenv("AUTOTRAIN_TOKEN")
16
  AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
17
  AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
18
 
 
 
 
 
 
19
 
20
  def main():
21
  logs_df = load_dataset("autoevaluate/evaluation-job-logs", use_auth_token=True, split="train").to_pandas()
22
- evaluated_projects_ds = load_dataset("autoevaluate/evaluated-project-ids", use_auth_token=True, split="train")
23
- projects_df = logs_df.copy()[(~logs_df["project_id"].isnull()) & (logs_df["is_evaluated"] == False)]
 
 
 
 
24
  projects_to_approve = projects_df["project_id"].astype(int).tolist()
25
 
26
  for project_id in projects_to_approve:
27
- project_status = http_get(
28
- path=f"/projects/{project_id}",
29
- token=HF_TOKEN,
30
- domain=AUTOTRAIN_BACKEND_API,
31
- ).json()
32
- if project_status["status"] == 3:
33
- train_job_resp = http_post(
34
- path=f"/projects/{project_id}/start_training",
35
  token=HF_TOKEN,
36
  domain=AUTOTRAIN_BACKEND_API,
37
  ).json()
38
- print(f"🏃‍♂️ Project {project_id} approval response: {train_job_resp}")
39
- # if train_job_resp["approved"] == True:
40
- # # Update evaluation status
 
 
 
 
 
 
 
41
 
42
 
43
  if __name__ == "__main__":
16
  AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
17
  AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
18
 
19
+ if "staging" in AUTOTRAIN_BACKEND_API:
20
+ AUTOTRAIN_ENV = "staging"
21
+ else:
22
+ AUTOTRAIN_ENV = "prod"
23
+
24
 
25
  def main():
26
  logs_df = load_dataset("autoevaluate/evaluation-job-logs", use_auth_token=True, split="train").to_pandas()
27
+ # Filter out
28
+ # - legacy AutoTrain submissions prior to project approvals was implemented
29
+ # - submissions for appropriate AutoTrain environment (staging vs prod)
30
+ projects_df = logs_df.copy()[
31
+ (~logs_df["project_id"].isnull()) & (logs_df.query(f"autotrain_env == '{AUTOTRAIN_ENV}'"))
32
+ ]
33
  projects_to_approve = projects_df["project_id"].astype(int).tolist()
34
 
35
  for project_id in projects_to_approve:
36
+ try:
37
+ project_info = http_get(
38
+ path=f"/projects/{project_id}",
 
 
 
 
 
39
  token=HF_TOKEN,
40
  domain=AUTOTRAIN_BACKEND_API,
41
  ).json()
42
+ if project_info["status"] == 3 and project_info["training_status"] == "not_started":
43
+ train_job_resp = http_post(
44
+ path=f"/projects/{project_id}/start_training",
45
+ token=HF_TOKEN,
46
+ domain=AUTOTRAIN_BACKEND_API,
47
+ ).json()
48
+ print(f"🏃‍♂️ Project {project_id} approval response: {train_job_resp}")
49
+ except:
50
+ print(f"There was a problem obtaining the project info for project ID {project_id}")
51
+ pass
52
 
53
 
54
  if __name__ == "__main__":