Merge branch 'main' of hf.co:spaces/openadmet/OpenADMET_Challenge_Demo

about.py

@@ -31,8 +31,10 @@ multiplier_dict = {"LogD": 1,
 
 TOKEN = os.environ.get("HF_TOKEN")
 CACHE_PATH=os.getenv("HF_HOME", ".")
+THROTTLE_MINUTES =   480 # minutes between submissions
 API = HfApi(token=TOKEN)
 organization="OpenADMET"
-submissions_repo = f'{organization}/openadmet-challenge-submissions' # private
-results_repo = f'{organization}/openadmet-challenge-results' # public
-test_repo = f'{organization}/openadmet-challenge-test-data' # private
+submissions_repo = f'{organization}/openadmet-expansionrx-challenge-submissions' # private
+results_repo_test = f'{organization}/openadmet-expansionrx-challenge-results' # public
+results_repo_validation = f'{organization}/openadmet-expansionrx-challenge-results-validation' # public
+test_repo = f'{organization}/openadmet-expansionrx-challenge-test-data' # private

app.py

@@ -10,13 +10,17 @@ from utils import (
     fetch_dataset_df,
     map_metric_to_stats,
 )
-
+from datasets import load_dataset
+import tempfile
+from loguru import logger
 from about import ENDPOINTS, LB_COLS, LB_AVG, LB_DTYPES
 
 
+
 ALL_EPS = ['Average'] + ENDPOINTS
 
 def build_leaderboard(df_results):
+    logger.info("Rebuilding leaderboard data...")
     per_ep = {}
     for ep in ALL_EPS:
         df = df_results[df_results["Endpoint"] == ep].copy()
@@ -43,7 +47,7 @@ def build_leaderboard(df_results):
             sorted_df = df.sort_values(by="mean_MAE", ascending=True, kind="stable")
             sorted_df = map_metric_to_stats(sorted_df)
             per_ep[ep] = sorted_df[LB_COLS]
-
+    logger.info("Finished rebuilding leaderboard data.")
     return per_ep
     
 # Initialize global dataframe
@@ -53,8 +57,8 @@ def gradio_interface():
 
     with gr.Blocks(title="OpenADMET ADMET Challenge", fill_height=False,
                    theme=gr.themes.Default(text_size=sizes.text_lg)) as demo:
-        timer = gr.Timer(15)  # Run every 15 seconds when page is focused
-        data_version = gr.State(0)  # Track data changes  
+        timer = gr.Timer(30)  # Run every 30 seconds when page is focused
+        data_version = gr.State(0)  # Track data changes
         def update_current_dataframe(v):
             global current_df
             new_df = fetch_dataset_df()
@@ -105,7 +109,7 @@ def gradio_interface():
 
         **OpenADMET** aims to address these challenges through an open science effort to build predictive models of ADMET properties by characterizing the proteins and mechanisms 
         that give rise to these properties through integrated structural biology, high throughput experimentation and integrative computational models. 
-        Read more about our strategy to transform drug discovery on our [website](https://openadmet.org/community/blogs/whatisopenadmet/). 
+        Read more about our strategy to transform drug discovery on our [website](https://openadmet.ghost.io/what-is-openadmet/). 
 
         Critical to our mission is developing open datasets and running community blind challenges to assess the current state of the art in ADMET modeling. 
         Building on the sucess of the recent [ASAP-Polaris-OpenADMET blind challenge](https://chemrxiv.org/engage/chemrxiv/article-details/68ac00d1728bf9025e22fe45) in computational methods for drug discovery,
@@ -128,7 +132,7 @@ def gradio_interface():
         - Mouse Brain Protein Binding (**MBPB**): % Unbound
         - Mouse Gastrocnemius Muscle Binding (**MGMB**): % Unbound
 
-        Find more information about these endpoints on our [blog](https://openadmet.org/community/blogs/challenge_announcement2/).
+        Find more information about these endpoints on our [blog](https://openadmet.ghost.io/openadmet-expansionrx-blind-challenge/).
 
         ## ✅ How to Participate
         1. **Register**: Create an account with Hugging Face.
@@ -157,6 +161,9 @@ def gradio_interface():
 
         You can download the training data from the [Hugging Face dataset](https://huggingface.co/datasets/openadmet/openadmet-challenge-train-data).
         The test set will remained blinded until the challenge submission deadline. You will be tasked with predicting the same set of ADMET endpoints for the test set molecules.
+        
+        The training and blinded test set will also be made available on the [CDD Vault](https://www.collaborativedrug.com/). An account to access the CDD Vault can be requested by emailing **openadmet@omsf.io**. 
+        Note that by joining the Vault, your account will be visible to other participants, so this option is **not recommended for those wishing to remain anonymous.**
 
         ## 📝 Evaluation
         The challenge will be judged based on the following criteria:
@@ -172,12 +179,16 @@ def gradio_interface():
 
         📅 **Timeline**:  
         - **September 16:** Challenge announcement
-        - **October XX:** Second announcement and sample data release
+        - **October 14:** Second announcement and sample data release
         - **October 27:** Challenge starts
         - **October-November:** Online Q&A sessions and support via the Discord channel
         - **January 19, 2026:** Submission closes
         - **January 26, 2026:** Winners announced
 
+        ## Acknowledgements
+        We gratefully acknowledge Jon Ainsley, Andrew Good, Elyse Bourque, Lakshminarayana Vogeti, Renato Skerlj, Tiansheng Wang, and Mark Ledeboer for generously
+        providing the Expansion Therapeutics dataset used in this challenge as an in-kind contribution.
+
         ---
 
         """
@@ -242,7 +253,7 @@ def gradio_interface():
                         select_columns=LB_AVG,
                         search_columns=["user"],
                         render=True,
-                        every=15,
+                        every=30,
                     )
                 # per-endpoint leaderboard
                 for endpoint in ENDPOINTS:
@@ -253,7 +264,7 @@ def gradio_interface():
                             select_columns=LB_COLS,
                             search_columns=["user"],
                             render=True,
-                            every=15,
+                            every=30,
                         )
                 # Auto-refresh 
                 def refresh_if_changed():
@@ -346,11 +357,15 @@ def gradio_interface():
                             Only your latest submission will be considered.
 
                             Download a CSV file with the compounds in the test set here:
+
+                            **NOTE: Submission can sometimes take a few minutes to process**
+                            **Please be patient and wait for the status message to update and your submission to reach the leaderboard.**
                             """
                         )
+
                         download_btn = gr.DownloadButton(
                             label="📥 Download Test Set Compounds",
-                            value="data/expansion_data_test_blinded.csv",
+                            value="./data/expansion_data_test_blinded.csv",
                             variant="secondary",
                             )
                     with gr.Column():
@@ -392,4 +407,6 @@ def gradio_interface():
     return demo
 
 if __name__ == "__main__":
-    gradio_interface().launch(ssr_mode=False)
+    logger.info("Starting Gradio app...")
+    gradio_interface().launch(ssr_mode=False)
+    logger.info("Gradio app closed.")

evaluate.py

@@ -5,11 +5,12 @@ from typing import Optional
 from about import (
     ENDPOINTS, API, 
     submissions_repo, 
-    results_repo, 
+    results_repo_test, 
+    results_repo_validation, 
     test_repo,
-    multiplier_dict,
+    THROTTLE_MINUTES
 )
-from utils import bootstrap_metrics, convert_to_log
+from utils import bootstrap_metrics, clip_and_log_transform, fetch_dataset_df
 from huggingface_hub import hf_hub_download
 import datetime
 import io
@@ -22,6 +23,7 @@ from pydantic import (
     field_validator, 
     ValidationError
 )
+from loguru import logger
 
 HF_USERNAME_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-_]{1,38})$")
 def _safeify_username(username: str) -> str:
@@ -107,6 +109,16 @@ def submit_data(predictions_file: str,
     if user_state is None:
         raise gr.Error("Username or alias is required for submission.")
     
+
+    # check the last time the user submitted
+    data = fetch_dataset_df()
+    if not data[data['user'] == user_state].empty:
+        last_time = data[data['user'] == user_state]['submission time'].max()
+        delta = datetime.datetime.now(datetime.timezone.utc) - last_time.to_pydatetime()
+        if delta < datetime.timedelta(minutes=THROTTLE_MINUTES):
+            raise gr.Error(f"You have submitted within the last {THROTTLE_MINUTES} minutes. Please wait {THROTTLE_MINUTES - int(delta.total_seconds() // 60)} minutes before submitting again.")
+
+
     file_path = Path(predictions_file).resolve()
     if not file_path.exists():
         raise gr.Error("Uploaded file object does not have a valid file path.")
@@ -178,6 +190,15 @@ def submit_data(predictions_file: str,
     return "✅ Your submission has been received! Your scores will appear on the leaderboard shortly.", destination_csv
 
 def evaluate_data(filename: str) -> None:
+    # do test set first as a more stringent check of the submission w.r.t matching molecules
+    logger.info(f"Evaluating submission file {filename}")
+    # evaluate on the test set
+    _evaluate_data(filename, test_repo=test_repo, split_filename="data/expansion_data_test.csv", results_repo=results_repo_test)
+    # evaluate on the validation set
+    _evaluate_data(filename, test_repo=test_repo, split_filename="data/expansion_data_test_validation.csv", results_repo=results_repo_validation)
+    logger.info(f"Finished evaluating submission file {filename}")
+
+def _evaluate_data(filename: str, test_repo: str, split_filename: str, results_repo: str) -> None:
 
     # Load the submission csv
     try:
@@ -194,7 +215,7 @@ def evaluate_data(filename: str) -> None:
         test_path = hf_hub_download(
             repo_id=test_repo,
             repo_type="dataset",
-            filename="data/expansion_data_test.csv", 
+            filename=split_filename
         )
     except Exception as e:
         raise gr.Error(f"Failed to download test file: {e}")
@@ -252,6 +273,7 @@ def evaluate_data(filename: str) -> None:
     Path(tmp_name).unlink()
 
 
+
 def calculate_metrics(
         results_dataframe: pd.DataFrame,
         test_dataframe: pd.DataFrame
@@ -263,60 +285,75 @@ def calculate_metrics(
     # 1) Check all columns are present
     _check_required_columns(results_dataframe, "Results file", ["Molecule Name"] + ENDPOINTS)
     _check_required_columns(test_dataframe, "Test file", ["Molecule Name"] + ENDPOINTS)
-    # 2) Check all Molecules in the test set are present in the predictions
-    merged_df = pd.merge(test_dataframe, results_dataframe, on=['Molecule Name'], how='left', indicator=True)
-    if not (merged_df['_merge'] == 'both').all():
-        raise gr.Error("The predictions file is missing some molecules present in the test set. Please ensure all molecules are included.")
-    # TODO: What to do when a molecule is duplicated in the Predictions file?
+    
+    
+       # 2) Check all Molecules in the test set are present in the predictions
+    if not (test_dataframe['Molecule Name']).isin(results_dataframe['Molecule Name']).all():
+        raise gr.Error("Some molecules in the test set are missing from the predictions file. Please ensure all molecules are included.")
+
+
+    # 3) check no duplicated molecules in the predictions file
+    if results_dataframe['Molecule Name'].duplicated().any():
+        raise gr.Error("The predictions file contains duplicated molecules. Please ensure each molecule is only listed once.")
+    
+    # 4) Merge dataframes to ensure alignment
+    merged_df = results_dataframe.merge(
+        test_dataframe,
+        on="Molecule Name",
+        suffixes=('_pred', '_true'),
+        how="inner"
+    )
+    merged_df = merged_df.sort_values("Molecule Name")
+
+    # 5) loop over endpoints 
 
-    # Compute leaderboard DataFrame
     final_cols = ["MAE", "RAE", "R2", "Spearman R", "Kendall's Tau"]
     all_endpoint_results = []
-    for i, measurement in enumerate(ENDPOINTS):  
-        df_pred = results_dataframe[['Molecule Name', measurement]].copy()
-        df_true = test_dataframe[['Molecule Name', measurement]].copy()
-        # coerce numeric columns
-        df_pred[measurement] = pd.to_numeric(df_pred[measurement], errors="coerce")
-        df_true[measurement] = pd.to_numeric(df_true[measurement], errors="coerce")
-
-        if df_pred[measurement].isnull().all():
-            # TODO: Allow missing endpoints or raise an error?
-            raise gr.Error(f"All predictions are missing for endpoint {measurement}. Please provide valid predictions.")
+
+    for ept in ENDPOINTS:
+        pred_col = f"{ept}_pred"
+        true_col = f"{ept}_true"
+
+        # cast to numeric, coerce errors to NaN
+        merged_df[pred_col] = pd.to_numeric(merged_df[pred_col], errors="coerce")
+        merged_df[true_col] = pd.to_numeric(merged_df[true_col], errors="coerce")
+
+        if merged_df[pred_col].isnull().all():
+            raise gr.Error(f"All predictions are missing for endpoint {ept}. Please provide valid predictions.")
         
-        # Drop NaNs and calculate coverage
-        merged = (
-            df_pred.rename(columns={measurement: f"{measurement}_pred"})
-                .merge(
-                    df_true.rename(columns={measurement: f"{measurement}_true"}),
-                    on="Molecule Name",
-                    how="inner",
-                )
-                .dropna(subset=[f"{measurement}_pred", f"{measurement}_true"])
-        )
-        merged = merged.sort_values("Molecule Name", kind="stable")
-        pred_col = f"{measurement}_pred"
-        true_col = f"{measurement}_true"
-
-        if measurement not in ['logD']:
-            # Force log scale for all endpoints except LogD (for outliers)
-            merged[pred_col] = convert_to_log(merged[pred_col], multiplier_dict.get(measurement, 1)).to_numpy()
-            merged[true_col]  = convert_to_log(merged[true_col], multiplier_dict.get(measurement, 1)).to_numpy()
-
-        y_pred = merged[pred_col].to_numpy()
-        y_true = merged[true_col].to_numpy()
-        # Calculate dataframe with the metrics for 1000 bootstraps
-        bootstrap_df = bootstrap_metrics(y_pred, y_true, measurement, n_bootstrap_samples=1000)
+        # subset and drop NaNs
+        subset = merged_df[[pred_col, true_col]].dropna()
+        if subset.empty:
+            raise gr.Error(f"No valid data available for endpoint {ept} after removing NaNs.")
+        
+        # extract numpy arrays
+        y_pred = subset[pred_col].to_numpy()
+        y_true = subset[true_col].to_numpy()
+
+        # apply log10 + 1 transform except for logD
+        if ept.lower() not in ['logd']:
+            y_true_log = clip_and_log_transform(y_true)
+            y_pred_log = clip_and_log_transform(y_pred)
+
+        else:
+            y_true_log = y_true
+            y_pred_log = y_pred
+
+        # calculate metrics with bootstrapping
+        bootstrap_df = bootstrap_metrics(y_pred_log, y_true_log, ept, n_bootstrap_samples=1000)
         df_endpoint = bootstrap_df.pivot_table(
             index=["Endpoint"],
             columns="Metric", 
             values="Value", 
             aggfunc=["mean", "std"]
         ).reset_index()
+
         # Get a df with columns 'mean_MAE', 'std_MAE', ...
         df_endpoint.columns = [
             f'{i}_{j}' if i != '' else j for i, j in df_endpoint.columns
         ]
-        df_endpoint.rename(columns={'Endpoint_': 'Endpoint'}, inplace=True) 
+
+        df_endpoint.rename(columns={'Endpoint_': 'Endpoint'}, inplace=True)
         all_endpoint_results.append(df_endpoint)
 
     df_results = pd.concat(all_endpoint_results, ignore_index=True)

requirements.txt

@@ -4,4 +4,5 @@ huggingface_hub
 gradio-leaderboard
 plotly
 scipy
-scikit-learn
+scikit-learn
+loguru

utils.py

@@ -3,8 +3,9 @@ import pandas as pd
 import numpy as np
 from typing import Tuple
 from datasets import load_dataset, Features, Value
-from about import results_repo
+from about import results_repo_validation, results_repo_test
 from about import METRICS, STANDARD_COLS
+from loguru import logger
 
 def make_user_clickable(name: str):
     link =f'https://huggingface.co/{name}'
@@ -13,7 +14,7 @@ def make_tag_clickable(tag: str):
     return f'<a target="_blank" href="{tag}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">link</a>'
 
 def fetch_dataset_df():
-
+    logger.info("Fetching latest results dataset from Hugging Face Hub...")
     # Specify feature types to load results dataset
     metric_features = {
         f'mean_{m}': Value('float64') for m in METRICS
@@ -30,7 +31,7 @@ def fetch_dataset_df():
     }
     feature_schema = Features(metric_features | other_features)
 
-    dset = load_dataset(results_repo, 
+    dset = load_dataset(results_repo_validation, # change to results_repo_test for test set
                         split='train', 
                         features=feature_schema,
                         download_mode="force_redownload")
@@ -57,11 +58,19 @@ def fetch_dataset_df():
     latest.rename(columns={"submission_time": "submission time"}, inplace=True)
     return latest
 
-def convert_to_log(data: pd.Series, multiplier: float) -> pd.Series:
-    # Add 0.01 to avoid inf 
-    values = np.clip(data, a_min=0.01, a_max=None)
-    values = values * multiplier # Adjust units
-    return np.log10(values)
+
+def clip_and_log_transform(y: np.ndarray):
+    """
+    Clip to a detection limit and transform to log10 scale.
+
+    Parameters
+    ----------
+    y : np.ndarray
+        The array to be clipped and transformed.
+    """
+    y = np.clip(y, a_min=0, a_max=None)
+    return np.log10(y + 1)
+
 
 def bootstrap_sampling(size: int, n_samples: int) -> np.ndarray:
     """
@@ -87,14 +96,12 @@ def metrics_per_ep(pred: np.ndarray,
                    true: np.ndarray
     )->Tuple[float, float, float, float]:
     """Predict evaluation metrics for a single sample
-
     Parameters
     ----------
     pred : np.ndarray
         Array with predictions
     true : np.ndarray
         Array with actual values
-
     Returns
     -------
     Tuple[float, float, float, float]
@@ -119,12 +126,11 @@ def metrics_per_ep(pred: np.ndarray,
     return mae, rae, r2, spr, ktau
 
 def bootstrap_metrics(pred: np.ndarray, 
-                      true: np.ndarray, 
-                      endpoint: str, 
+                      true: np.ndarray,
+                      endpoint: str,
                       n_bootstrap_samples=1000
     )->pd.DataFrame:
     """Calculate bootstrap metrics given predicted and true values
-
     Parameters
     ----------
     pred : np.ndarray
@@ -135,7 +141,6 @@ def bootstrap_metrics(pred: np.ndarray,
         String with endpoint
     n_bootstrap_samples : int, optional
         Size of bootstrapsample, by default 1000
-
     Returns
     -------
     pd.DataFrame