import glob
import json
import logging
import os
import re
import traceback
from datetime import datetime, timedelta, timezone
import gradio as gr
import yaml
from loguru import logger
from app_configs import DAILY_SUBMISSION_LIMIT_PER_USER
from display.formatting import styled_error, styled_message
from envs import API, EVAL_REQUESTS_PATH, EXAMPLES_PATH, QUEUE_REPO
from submission.structs import CompetitionType, Submission, SubmissionStatus
from workflows.structs import TossupWorkflow, Workflow
def get_user_submissions(username: str, competition_type: str, pattern: str = None) -> list[Submission]:
"""Get all submissions for a user."""
out_dir = f"{EVAL_REQUESTS_PATH}/{username}"
submissions = []
if not os.path.exists(out_dir):
return submissions
for file in os.listdir(out_dir):
if not file.startswith(f"{competition_type}_"):
continue
if pattern is not None and pattern not in file:
continue
try:
with open(os.path.join(out_dir, file), "r") as f:
submission = Submission.from_dict(json.load(f))
submissions.append(submission)
except Exception as e:
logger.error(f"Error loading submission {file}: {e}")
return submissions
def get_user_submission_names(competition_type: str, profile: gr.OAuthProfile | None) -> list[str]:
"""Get all submission model names for a user."""
if profile is None:
logger.warning("No user profile provided. Returning empty list.")
return []
submissions = get_user_submissions(profile.username, competition_type)
return [f"{s.username}/{s.model_name}" for s in submissions]
def get_demo_example_submissions(competition_type: str) -> list[str]:
"""Get all submissions for a demo example."""
examples_dir = f"{EXAMPLES_PATH}/{competition_type}"
return [f"umdclip/{os.path.basename(f).removesuffix('.yaml')}" for f in glob.glob(f"{examples_dir}/*.yaml")]
def get_user_submissions_by_date(
username: str, competition_type: str, year: int, month: int, day: int
) -> dict[str, list[Submission]]:
"""Get all submissions for a user for a given competition type."""
date_str = datetime(year, month, day).strftime("%Y%m%d")
out_dir = f"{EVAL_REQUESTS_PATH}/{username}"
if not os.path.exists(out_dir):
return {}
submissions = []
for file in os.listdir(out_dir):
if file.startswith(f"{competition_type}_") and date_str in file:
try:
submission = Submission.from_dict(json.load(open(os.path.join(out_dir, file))))
submissions.append(submission)
except Exception as e:
logger.exception(e)
logger.warning(f"Error loading submission {file}: {e}")
return submissions
def get_user_submissions_today(username: str, competition_type: str) -> list[Submission]:
"""Get all submissions for a user for a given competition type."""
today = datetime.now(timezone.utc)
return get_user_submissions_by_date(username, competition_type, today.year, today.month, today.day)
def get_time_until_next_submission(tz: timezone = timezone.utc) -> str:
next_day_00 = datetime.now(tz) + timedelta(days=1)
next_day_00 = next_day_00.replace(hour=0, minute=0, second=0, microsecond=0)
remaining_time = next_day_00 - datetime.now(tz)
hours = remaining_time.seconds // 3600
minutes = (remaining_time.seconds % 3600) // 60
remaining_time_str = f"{hours} hours {minutes} mins"
return remaining_time_str
def create_submission(
username: str,
model_name: str,
description: str,
workflow: Workflow,
competition_type: CompetitionType,
) -> Submission:
"""
Create a submission for a tossup model.
Args:
name: Display name of the submission
description: Detailed description of what the submission does
user_email: Email of the user who created the submission
workflow: The workflow configuration for the tossup model
Returns:
Submission object if successful, None if validation fails
"""
# Create the submission
dt = datetime.now(timezone.utc)
submission = Submission(
id=f"{competition_type}_{dt.strftime('%Y%m%d_%H%M%S')}_{model_name.lower().replace(' ', '_')}",
model_name=model_name,
username=username,
description=description,
competition_type=competition_type,
submission_type="simple_workflow",
workflow=workflow,
status="submitted",
created_at=dt.isoformat(),
updated_at=dt.isoformat(),
)
return submission
def validate_model_name(model_name: str):
# check if model_name has no white spaces, no special characters apart from _ and -
if " " in model_name:
return False, "Model name cannot contain spaces."
if not re.match(r"^[a-zA-Z0-9_-]+$", model_name):
return False, "Model name can only contain letters, numbers, underscores, and hyphens."
if not re.match(r"^[a-zA-Z]", model_name):
return False, "Model name must start with a letter."
return True, ""
def submit_model(
model_name: str,
description: str,
workflow: Workflow,
competition_type: CompetitionType,
profile: gr.OAuthProfile | None,
) -> str:
"""
Submit a tossup model for evaluation.
Args:
name: Display name of the submission
description: Detailed description of what the submission does
user_email: Email of the user who created the submission
workflow: The workflow configuration for the tossup model
Returns:
Status message
"""
if profile is None:
return styled_error("Authentication required. Please log in first to submit your model.")
username = profile.username
if len(get_user_submissions_today(username, competition_type)) >= DAILY_SUBMISSION_LIMIT_PER_USER:
time_str = get_time_until_next_submission()
return styled_error(
f"Daily submission limit of {DAILY_SUBMISSION_LIMIT_PER_USER} reached. Please try again in \n {time_str}."
)
if f"{username}/{model_name}" in get_user_submission_names(competition_type, profile):
return styled_error(
f"Submission Error!
'{model_name}' already exists. Please use a different name for your submission."
)
is_valid, error_msg = validate_model_name(model_name)
if not is_valid:
return styled_error(f"Submission Error! Invalid model name '{model_name}'.
{error_msg}")
try:
submission = create_submission(
username=username,
model_name=model_name,
description=description,
workflow=workflow,
competition_type=competition_type,
)
# Convert to dictionary format
submission_dict = submission.to_dict()
# Create output directory path
out_dir = f"{EVAL_REQUESTS_PATH}/{username}"
out_path = f"{out_dir}/{submission.id}.json"
# Upload to HuggingFace dataset
API.upload_file(
path_or_fileobj=json.dumps(submission_dict, indent=2).encode(),
path_in_repo=out_path.split("eval-queue/")[1],
repo_id=QUEUE_REPO,
repo_type="dataset",
commit_message=f"Add tossup submission {submission.id}",
)
return styled_message(
f"Successfully submitted tossup model!
"
f"Submission name: {username}/{model_name}
"
f"Please wait for up to an hour for the model to show in the PENDING list."
)
except Exception as e:
traceback.print_exc()
return styled_error(f"Error submitting model: {str(e)}")
def load_demo_example(model_name: str, competition_type: CompetitionType) -> Workflow | TossupWorkflow:
"""Load a demo example submission."""
examples_dir = f"{EXAMPLES_PATH}/{competition_type}"
filepath = f"{examples_dir}/{model_name}.yaml"
if not os.path.exists(filepath):
raise ValueError(f"Demo example file {filepath} not found")
with open(filepath, "r") as f:
yaml_data = yaml.safe_load(f)
if competition_type == "tossup":
return TossupWorkflow.model_validate(yaml_data)
else:
return Workflow.model_validate(yaml_data)
def load_submission(model_name: str, competition_type: CompetitionType, profile: gr.OAuthProfile | None) -> Submission:
if profile is None:
logging.error("Authentication required. Please log in to view your submissions.")
return styled_error("Authentication required. Please log in to view your submissions.")
username = profile.username
submissions = get_user_submissions(username, competition_type, model_name)
if len(submissions) == 0:
return styled_error(f"Submission {model_name} not found.")
return submissions[0]
if __name__ == "__main__":
# Example usage
from workflows.factory import create_quizbowl_simple_step_initial_setup
# Create workflow
model_step = create_quizbowl_simple_step_initial_setup()
model_step.model = "gpt-4"
model_step.provider = "openai"
model_step.temperature = 0.7
workflow = Workflow(
inputs=["question_text"],
outputs={"answer": "A.answer", "confidence": "A.confidence"},
steps={"A": model_step},
)
# Submit model
result = submit_model(
model_name="GPT-4 Tossup",
description="A simple GPT-4 model for tossup questions",
workflow=workflow,
competition_type="tossup",
)
print(result)
# %%