✨ Submission handling and integration of results into results dataset added

.gitignore

@@ -12,3 +12,4 @@ eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/
+debugging_and_dev/

app.py

@@ -1,9 +1,11 @@
 from frontend.layout import create_main_interface
 from frontend.leaderboard import refresh_leaderboard
+from frontend.submission import handle_submission
 
 # Create the main interface with callbacks
 demo, leaderboard_table = create_main_interface(
-    refresh_callback=refresh_leaderboard
+    refresh_callback=refresh_leaderboard,
+    submit_callback=handle_submission
 )
 
 if __name__ == "__main__":

backend/api.py

@@ -0,0 +1,47 @@
+import asyncio
+from typing import Dict, Any
+from .submission import process_submission
+
+
+# Sample data - replace with actual data loading
+SAMPLE_SMILES = ["CCO", "c1ccccc1", "CC(=O)Cl", "C1=CC=CN=C1"]
+SAMPLE_LABELS = {
+    "CCO": {"NR-AR": 0.1, "NR-AR-LBD": 0.2},
+    "c1ccccc1": {"NR-AR": 0.3, "NR-AR-LBD": 0.4},
+    "CC(=O)Cl": {"NR-AR": 0.5, "NR-AR-LBD": 0.6},
+    "C1=CC=CN=C1": {"NR-AR": 0.7, "NR-AR-LBD": 0.8}
+}
+
+
+async def submit_model(
+    model_name: str,
+    hf_space_tag: str,
+    model_description: str,
+    publication_title: str = "",
+    publication_link: str = "",
+    model_size: str = "",
+    pretraining: str = "",
+    organization: str = ""
+) -> Dict[str, Any]:
+    """API endpoint for model submission."""
+    
+    record = await process_submission(
+        model_name=model_name,
+        hf_space_tag=hf_space_tag,
+        model_description=model_description,
+        publication_title=publication_title,
+        publication_link=publication_link,
+        model_size=model_size,
+        pretraining=pretraining,
+        organization=organization,
+        smiles_list=SAMPLE_SMILES,
+        true_labels=SAMPLE_LABELS
+    )
+    
+    return record
+
+
+def get_submission_status(submission_id: str) -> Dict[str, str]:
+    """Get the status of a submission."""
+    # Placeholder - implement proper status tracking
+    return {"status": "completed", "message": "Submission processed successfully"}

backend/data_loader.py

@@ -28,19 +28,23 @@ def load_leaderboard_data() -> pd.DataFrame:
     else:
         raise ValueError("Dataset does not contain a 'test' split.")
     
-    # Convert to DataFrame
+    # Convert to DataFrame (new schema only)
     rows = []
     for entry in results_data:
         config = entry['config']
         results = entry['results']
         
+        # Only include approved entries
+        if not config.get('approved', False):
+            continue
+            
         # Create a row with all the data
         row = {
             'Model': config['model_name'],
-            'Model Description': config['model_description'],
-            'Publication': config['publication_title'],
-            'Parameters': config['parameters'],
-            'Date Added': str(config['date_added']).split()[0],  # Just the date part
+            'Model Description': config['model_description'], 
+            'Publication': config.get('publication_title', ''),
+            'Parameters': config.get('model_size', ''),
+            'Date Added': str(config.get('date_approved', config.get('date_submitted', ''))).split()[0],
             'Overall Score': results['overall_score']['roc_auc']
         }
         
@@ -52,6 +56,18 @@ def load_leaderboard_data() -> pd.DataFrame:
         rows.append(row)
     
     df = pd.DataFrame(rows)
+    
+    # Handle empty dataset case
+    if df.empty:
+        print("No approved submissions found. Creating empty DataFrame with proper columns.")
+        # Create empty DataFrame with expected columns
+        columns = [
+            'Model', 'Model Description', 'Publication', 'Parameters', 'Date Added', 'Overall Score',
+            'NR-AR', 'NR-AR-LBD', 'NR-AhR', 'NR-Aromatase', 'NR-ER', 'NR-ER-LBD', 
+            'NR-PPAR-gamma', 'SR-ARE', 'SR-ATAD5', 'SR-HSE', 'SR-MMP', 'SR-p53'
+        ]
+        df = pd.DataFrame(columns=columns)
+    
     print(df)
     print(f"Created DataFrame with shape: {df.shape}")
     return df

backend/dataset_storage.py

@@ -0,0 +1,66 @@
+from datasets import Dataset, load_dataset
+from typing import Dict, Any, List
+from config.settings import RESULTS_DATASET, HF_TOKEN
+
+
+def save_submission_to_dataset(submission_record: Dict[str, Any]) -> None:
+    """Save a submission record to the HuggingFace dataset."""
+    
+    # Load existing dataset or create new one
+    try:
+        dataset = load_dataset(RESULTS_DATASET, token=HF_TOKEN, split="test")
+        existing_data = list(dataset)
+    except:
+        # Dataset doesn't exist or is empty, start fresh
+        existing_data = []
+    
+    # Add new submission
+    existing_data.append(submission_record)
+    
+    # Create new dataset
+    new_dataset = Dataset.from_list(existing_data)
+    
+    # Push to HuggingFace
+    new_dataset.push_to_hub(
+        RESULTS_DATASET, 
+        token=HF_TOKEN,
+        split="test"
+    )
+
+
+def get_all_submissions(approved_only: bool = False) -> List[Dict[str, Any]]:
+    """Get all submissions from the dataset."""
+    
+    dataset = load_dataset(RESULTS_DATASET, token=HF_TOKEN, split="test")
+    submissions = list(dataset)
+    
+    if approved_only:
+        submissions = [s for s in submissions if s["config"]["approved"]]
+    
+    return submissions
+
+
+def update_submission_approval(model_name: str, hf_space_tag: str, approved: bool) -> None:
+    """Update the approval status of a submission."""
+    
+    # Load dataset
+    dataset = load_dataset(RESULTS_DATASET, token=HF_TOKEN, split="test")
+    data = list(dataset)
+    
+    # Find and update the submission
+    for submission in data:
+        config = submission["config"]
+        if config["model_name"] == model_name and config["hf_space_tag"] == hf_space_tag:
+            config["approved"] = approved
+            if approved:
+                from datetime import datetime
+                config["date_approved"] = datetime.now().isoformat()
+            break
+    
+    # Save updated dataset
+    updated_dataset = Dataset.from_list(data)
+    updated_dataset.push_to_hub(
+        RESULTS_DATASET,
+        token=HF_TOKEN, 
+        split="test"
+    )

backend/evaluator.py

@@ -0,0 +1,52 @@
+import asyncio
+import httpx
+from typing import List, Dict, Any
+
+BATCH_SIZE = 1000
+TIMEOUT_S = 600
+MAX_RETRIES = 3
+RETRY_DELAY = 1
+
+
+def chunks(xs: List[str], n: int):
+    for i in range(0, len(xs), n):
+        yield xs[i:i+n]
+
+
+async def fetch_metadata(client: httpx.AsyncClient, base_url: str) -> Dict[str, Any]:
+    for attempt in range(MAX_RETRIES):
+        r = await client.get(f"{base_url}/metadata", timeout=30)
+        r.raise_for_status()
+        return r.json()
+
+
+async def call_predict(client: httpx.AsyncClient, base_url: str, smiles_batch: List[str]) -> Dict[str, Any]:
+    for attempt in range(MAX_RETRIES):
+        r = await client.post(
+            f"{base_url}/predict",
+            json={"smiles": smiles_batch},
+            timeout=TIMEOUT_S,
+        )
+        r.raise_for_status()
+        return r.json()
+
+
+async def evaluate_model(hf_space_tag: str, smiles_list: List[str]) -> Dict[str, Any]:
+    # Convert username/space-name to username-space-name.hf.space
+    base_url = f"https://{hf_space_tag.replace('/', '-').replace('_', '-').lower()}.hf.space"
+    results = []
+    
+    async with httpx.AsyncClient() as client:
+        meta = await fetch_metadata(client, base_url)
+        max_bs = min(meta.get("max_batch_size", BATCH_SIZE), BATCH_SIZE)
+
+        for batch in chunks(smiles_list, max_bs):
+            resp = await call_predict(client, base_url, batch)
+            predictions_dict = resp["predictions"]
+            for smiles in batch:
+                if smiles in predictions_dict:
+                    results.append({"smiles": smiles, "raw_predictions": predictions_dict[smiles]})
+                else:
+                    results.append({"smiles": smiles, "raw_predictions": {}, "error": "No prediction found"})
+    
+    return {"results": results, "metadata": meta}

backend/schema.py

@@ -0,0 +1,63 @@
+from datetime import datetime
+from typing import Dict, List, Any, Optional
+
+
+def create_submission_record(
+    model_name: str,
+    hf_space_tag: str,
+    model_description: str,
+    publication_title: str,
+    publication_link: str,
+    model_size: str,
+    pretraining: str,
+    organization: str,
+    raw_predictions: List[Dict[str, Any]],
+    computed_metrics: Dict[str, Any],
+    status: str = "completed",
+    approved: bool = False
+) -> Dict[str, Any]:
+    """Create a standardized submission record for the HuggingFace dataset."""
+    
+    now = datetime.now().isoformat()
+    
+    return {
+        "config": {
+            "model_name": model_name,
+            "hf_space_tag": hf_space_tag,
+            "model_description": model_description,
+            "publication_title": publication_title,
+            "publication_link": publication_link,
+            "model_size": model_size,
+            "pretraining": pretraining,
+            "organization": organization,
+            "date_submitted": now,
+            "date_approved": None,
+            "status": status,
+            "approved": approved
+        },
+        "raw_predictions": raw_predictions,
+        "results": computed_metrics
+    }
+
+
+def get_dataset_schema() -> Dict[str, Any]:
+    """Return the HuggingFace dataset schema."""
+    
+    return {
+        "config": {
+            "model_name": "string",
+            "hf_space_tag": "string", 
+            "model_description": "string",
+            "publication_title": "string",
+            "publication_link": "string",
+            "model_size": "string",
+            "pretraining": "string",
+            "organization": "string",
+            "date_submitted": "string",
+            "date_approved": "string",
+            "status": "string",
+            "approved": "bool"
+        },
+        "raw_predictions": "list",
+        "results": "dict"
+    }

backend/submission.py

@@ -0,0 +1,70 @@
+import asyncio
+from typing import Dict, Any, List
+from .evaluator import evaluate_model
+from .schema import create_submission_record
+from .dataset_storage import save_submission_to_dataset
+
+
+async def process_submission(
+    model_name: str,
+    hf_space_tag: str,
+    model_description: str,
+    publication_title: str,
+    publication_link: str,
+    model_size: str,
+    pretraining: str,
+    organization: str,
+    smiles_list: List[str],
+    true_labels: Dict[str, Dict[str, float]]
+) -> Dict[str, Any]:
+    """Process a complete submission from evaluation to metrics computation."""
+    
+    # Step 1: Evaluate the model
+    evaluation_result = await evaluate_model(hf_space_tag, smiles_list)
+    
+    # Step 2: Compute metrics
+    metrics = compute_metrics(evaluation_result["results"], true_labels)
+    
+    # Step 3: Create the submission record
+    record = create_submission_record(
+        model_name=model_name,
+        hf_space_tag=hf_space_tag,
+        model_description=model_description,
+        publication_title=publication_title,
+        publication_link=publication_link,
+        model_size=model_size,
+        pretraining=pretraining,
+        organization=organization,
+        raw_predictions=evaluation_result["results"],
+        computed_metrics=metrics,
+        status="completed",
+        approved=False
+    )
+    
+    # Step 4: Save to HuggingFace dataset
+    save_submission_to_dataset(record)
+    
+    return record
+
+
+def compute_metrics(predictions: List[Dict[str, Any]], true_labels: Dict[str, Dict[str, float]]) -> Dict[str, Any]:
+    """Compute evaluation metrics from predictions and true labels."""
+    
+    # Simple placeholder - you'll want to implement proper ROC-AUC computation
+    task_metrics = {}
+    
+    # Get all unique tasks from predictions
+    if predictions:
+        first_pred = predictions[0].get("raw_predictions", {})
+        tasks = list(first_pred.keys())
+        
+        for task in tasks:
+            # Placeholder metric computation
+            task_metrics[task] = {"roc_auc": 0.75}  # Replace with real computation
+    
+    # Overall score (average of task scores)
+    if task_metrics:
+        overall_score = sum(m["roc_auc"] for m in task_metrics.values()) / len(task_metrics)
+        task_metrics["overall_score"] = {"roc_auc": overall_score}
+    
+    return task_metrics

frontend/content.py

@@ -73,13 +73,48 @@ class SubmissionContent:
     
     title = "🚀 Submit Your Model"
     
+    form_labels = {
+        "model_name": "Model Name *",
+        "hf_space_tag": "HuggingFace Space Tag *", 
+        "model_description": "Model Description *",
+        "organization": "Organization",
+        "model_size": "Model Size",
+        "pretraining": "Pretraining Details",
+        "publication_title": "Publication Title",
+        "publication_link": "Publication Link"
+    }
+    
+    form_placeholders = {
+        "model_name": "e.g., AwesomeTox",
+        "hf_space_tag": "e.g., username/model-name", 
+        "model_description": "Brief description of your model architecture and approach...",
+        "organization": "e.g., University of Example",
+        "model_size": "e.g., 150M parameters",
+        "pretraining": "e.g., ChEMBL 29, ZINC-15",
+        "publication_title": "Title of associated paper",
+        "publication_link": "https://arxiv.org/abs/..."
+    }
+    
+    form_info = {
+        "model_name": "A short, descriptive name for your model",
+        "hf_space_tag": "Your HuggingFace space in format: username/space-name",
+        "model_description": "Describe your model, methodology, and key features"
+    }
+    
     @staticmethod
     def get_instructions_html() -> str:
         """Generate submission instructions HTML"""
-        return f"""
-        # TODO
-        add more text. Assumption: If content gets more, this tab will be displayed 
-        more nicely.
+        return """
+        <div class="instructions-section">
+            <p>Submit your HuggingFace space for evaluation on the Tox21 benchmark.</p>
+            <p><strong>Requirements:</strong></p>
+            <ul>
+                <li>Your space must implement <code>/metadata</code> and <code>/predict</code> endpoints</li>
+                <li>The <code>/predict</code> endpoint should accept a JSON payload with <code>{"smiles": [list_of_smiles]}</code></li>
+                <li>Response should be <code>{"predictions": {smiles: {task: score, ...}, ...}}</code></li>
+            </ul>
+            <p><em>* Required fields</em></p>
+        </div>
         """
 
 

frontend/layout.py

@@ -66,46 +66,69 @@ def create_submission_tab(submit_callback: Callable = None) -> gr.TabItem:
         # Instructions
         instructions_html = gr.HTML(SubmissionContent.get_instructions_html())
         
-        # Future form components (commented out for now)
-        """
+        # Submission form
         with gr.Group():
+            # Required fields
             model_name = gr.Textbox(
                 label=SubmissionContent.form_labels["model_name"],
-                placeholder=SubmissionContent.form_placeholders["model_name"]
+                placeholder=SubmissionContent.form_placeholders["model_name"],
+                info=SubmissionContent.form_info["model_name"]
             )
             
-            with gr.Row():
-                model_revision = gr.Textbox(
-                    label=SubmissionContent.form_labels["model_revision"],
-                    placeholder=SubmissionContent.form_placeholders["model_revision"],
-                    value="main"
-                )
-                model_type = gr.Dropdown(
-                    choices=SubmissionContent.model_types,
-                    label=SubmissionContent.form_labels["model_type"]
-                )
-                precision = gr.Dropdown(
-                    choices=SubmissionContent.precisions,
-                    label=SubmissionContent.form_labels["precision"],
-                    value="float16"
-                )
+            hf_space_tag = gr.Textbox(
+                label=SubmissionContent.form_labels["hf_space_tag"],
+                placeholder=SubmissionContent.form_placeholders["hf_space_tag"],
+                info=SubmissionContent.form_info["hf_space_tag"]
+            )
             
-            description = gr.Textbox(
-                label=SubmissionContent.form_labels["description"],
-                placeholder=SubmissionContent.form_placeholders["description"],
+            model_description = gr.Textbox(
+                label=SubmissionContent.form_labels["model_description"],
+                placeholder=SubmissionContent.form_placeholders["model_description"],
+                info=SubmissionContent.form_info["model_description"],
                 lines=3
             )
             
+            # Optional fields in accordion
+            with gr.Accordion("Additional Information (Optional)", open=False):
+                organization = gr.Textbox(
+                    label=SubmissionContent.form_labels["organization"],
+                    placeholder=SubmissionContent.form_placeholders["organization"]
+                )
+                
+                with gr.Row():
+                    model_size = gr.Textbox(
+                        label=SubmissionContent.form_labels["model_size"],
+                        placeholder=SubmissionContent.form_placeholders["model_size"]
+                    )
+                    pretraining = gr.Textbox(
+                        label=SubmissionContent.form_labels["pretraining"],
+                        placeholder=SubmissionContent.form_placeholders["pretraining"]
+                    )
+                
+                publication_title = gr.Textbox(
+                    label=SubmissionContent.form_labels["publication_title"],
+                    placeholder=SubmissionContent.form_placeholders["publication_title"]
+                )
+                
+                publication_link = gr.Textbox(
+                    label=SubmissionContent.form_labels["publication_link"],
+                    placeholder=SubmissionContent.form_placeholders["publication_link"]
+                )
+            
+            # Submit button and result
             submit_btn = gr.Button("Submit Model", variant="primary")
             result_msg = gr.HTML()
             
             if submit_callback:
                 submit_btn.click(
                     fn=submit_callback,
-                    inputs=[model_name, model_revision, model_type, precision, description],
+                    inputs=[
+                        model_name, hf_space_tag, model_description,
+                        organization, model_size, pretraining,
+                        publication_title, publication_link
+                    ],
                     outputs=result_msg
                 )
-        """
     
     return tab
 

frontend/submission.py

@@ -0,0 +1,36 @@
+import asyncio
+import gradio as gr
+from backend.api import submit_model
+
+
+def handle_submission(
+    model_name, hf_space_tag, model_description,
+    organization, model_size, pretraining, 
+    publication_title, publication_link
+):
+    """Handle model submission from the form."""
+    
+    # Basic validation
+    if not model_name or not hf_space_tag or not model_description:
+        return "<div style='color: red;'>L Error: Please fill in all required fields (*)</div>"
+    
+    if "/" not in hf_space_tag:
+        return "<div style='color: red;'>L Error: HuggingFace space tag should be in format 'username/space-name'</div>"
+    
+    # Process submission
+    try:
+        result = asyncio.run(submit_model(
+            model_name=model_name,
+            hf_space_tag=hf_space_tag,
+            model_description=model_description,
+            organization=organization or "",
+            model_size=model_size or "",
+            pretraining=pretraining or "",
+            publication_title=publication_title or "",
+            publication_link=publication_link or ""
+        ))
+        
+        return "<div style='color: green;'> Success! Your model has been submitted for evaluation. Results pending approval.</div>"
+    
+    except Exception as e:
+        return f"<div style='color: red;'>L Error: {str(e)}</div>"

raw_predictions.json

@@ -0,0 +1,75 @@
+[
+  {
+    "participant": "RandomClassifiers",
+    "results": [
+      {
+        "smiles": "CCO",
+        "raw_predictions": {
+          "NR-AR": 0.3917455619722391,
+          "NR-AR-LBD": 0.26008200862052433,
+          "NR-AhR": 0.2101509144439867,
+          "NR-Aromatase": 0.2630385047554743,
+          "NR-ER": 0.7002048747849454,
+          "NR-ER-LBD": 0.15377240592596264,
+          "NR-PPAR-gamma": 0.1353658553753373,
+          "SR-ARE": 0.2797094295476972,
+          "SR-ATAD5": 0.8406408791404627,
+          "SR-HSE": 0.9222532746585554,
+          "SR-MMP": 0.7170385993040889,
+          "SR-p53": 0.19909124452286608
+        }
+      },
+      {
+        "smiles": "c1ccccc1",
+        "raw_predictions": {
+          "NR-AR": 0.6967615243164884,
+          "NR-AR-LBD": 0.5429334657310607,
+          "NR-AhR": 0.8359421675060189,
+          "NR-Aromatase": 0.21863713829746834,
+          "NR-ER": 0.1384164897724609,
+          "NR-ER-LBD": 0.0567773996289892,
+          "NR-PPAR-gamma": 0.884469677268269,
+          "SR-ARE": 0.9507655336335464,
+          "SR-ATAD5": 0.861007589725987,
+          "SR-HSE": 0.9886291113232994,
+          "SR-MMP": 0.6429608178936979,
+          "SR-p53": 0.3907555716000315
+        }
+      },
+      {
+        "smiles": "CC(=O)Cl",
+        "raw_predictions": {
+          "NR-AR": 0.7653254677767931,
+          "NR-AR-LBD": 0.47398075844041376,
+          "NR-AhR": 0.0364769256514208,
+          "NR-Aromatase": 0.6524090139294707,
+          "NR-ER": 0.6020335714581855,
+          "NR-ER-LBD": 0.4920265461047544,
+          "NR-PPAR-gamma": 0.2534192957556971,
+          "SR-ARE": 0.177969901329171,
+          "SR-ATAD5": 0.09011487142758401,
+          "SR-HSE": 0.9120474797445164,
+          "SR-MMP": 0.7510410188649429,
+          "SR-p53": 0.7914294317144421
+        }
+      },
+      {
+        "smiles": "C1=CC=CN=C1",
+        "raw_predictions": {
+          "NR-AR": 0.5975434925069701,
+          "NR-AR-LBD": 0.2177384387523179,
+          "NR-AhR": 0.5091539966886376,
+          "NR-Aromatase": 0.5279716247658367,
+          "NR-ER": 0.37037003396449375,
+          "NR-ER-LBD": 0.042919826736678934,
+          "NR-PPAR-gamma": 0.6182258605984476,
+          "SR-ARE": 0.9229999554560413,
+          "SR-ATAD5": 0.1531615980022496,
+          "SR-HSE": 0.08691301573299948,
+          "SR-MMP": 0.21204819276958253,
+          "SR-p53": 0.08065030781214522
+        }
+      }
+    ]
+  }
+]