Repository: Project-MONAI/model-zoo
Pull request #740: [part 1] Update all bundles to Huggingface and fix missing dependencies
Author: login: yiheng-wang-nv
Created: 2025-03-24T08:48:57Z | Merged: 2025-03-26T08:12:18Z

Part of #739 

### Description
This PR is used to:
1. update a part of bundle versions to enable huggingface hosting.
2. update premerge CI test scripts to resolve dependency conflicts (for example, `requirements-dev.txt` has conflicts with some bundles' requirements)
3. fix missing dependencies issues on some bundles

### Status
**Ready**

### Please ensure all the checkboxes:
<!--- Put an `x` in all the boxes that apply, and remove the not applicable items -->
- [x] Codeformat tests passed locally by running `./runtests.sh --codeformat`.
- [ ] In-line docstrings updated.
- [ ] Update `version` and `changelog` in `metadata.json` if changing an existing bundle.
- [ ] Please ensure the naming rules in config files meet our requirements (please refer to: `CONTRIBUTING.md`).
- [ ] Ensure versions of packages such as `monai`, `pytorch` and `numpy` are correct in `metadata.json`.
- [ ] Descriptions should be consistent with the content, such as `eval_metrics` of the provided weights and TorchScript modules.
- [ ] Files larger than 25MB are excluded and replaced by providing download links in `large_file.yml`.
- [ ] Avoid using path that contains personal information within config files (such as use `/home/your_name/` for `"bundle_root"`).



Diff:
diff --git a/.gitignore b/.gitignore
index 0f2765e6..c4c5b59f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -131,4 +131,5 @@ temp/
 *.zip
 models/*/models/*
 models/*/output/*
+models/*/eval/*
 hf_models/*/eval/*
diff --git a/ci/bundle_custom_data.py b/ci/bundle_custom_data.py
index 6f20d112..30aaa2b9 100644
--- a/ci/bundle_custom_data.py
+++ b/ci/bundle_custom_data.py
@@ -58,7 +58,7 @@
 # This dict is used for our CI tests to install required dependencies that cannot be installed by `pip install` directly.
 # If a bundle has this kind of dependencies, please add the bundle name (key), and the path of the install script (value)
 # into the dict.
-install_dependency_dict = {"maisi_ct_generative": "ci/install_scripts/install_maisi_ct_generative_dependency.sh"}
+install_dependency_dict = {}
 
 # This list is used for our CI tests to determine whether a bundle supports TensorRT export. Related
 # test will be employed for bundles in the dict.
diff --git a/ci/download_latest_bundle.py b/ci/download_latest_bundle.py
index 14ea3581..847d0ae7 100644
--- a/ci/download_latest_bundle.py
+++ b/ci/download_latest_bundle.py
@@ -14,23 +14,14 @@
 import os
 
 from monai.bundle import download
-from utils import get_checksum, get_hash_func, get_latest_version, get_version_checksum
+from utils import get_latest_version
 
 
 def download_latest_bundle(bundle_name: str, models_path: str, download_path: str):
     model_info_path = os.path.join(models_path, "model_info.json")
     version = get_latest_version(bundle_name=bundle_name, model_info_path=model_info_path)
-    checksum = get_version_checksum(bundle_name=bundle_name, version=version, model_info_path=model_info_path)
     download(name=bundle_name, source="monaihosting", version=version, bundle_dir=download_path)
 
-    # verify checksum
-    hash_func = get_hash_func(hash_type="sha1")
-    bundle_zip_name = f"{bundle_name}_v{version}.zip"
-    downloaded_checksum = get_checksum(dst_path=os.path.join(download_path, bundle_zip_name), hash_func=hash_func)
-    if checksum != downloaded_checksum:
-        raise ValueError(f"Checksum of {bundle_zip_name} is not correct.")
-    print(f"Checksum of downloaded bundle {bundle_name} is correct.")
-
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="")
diff --git a/ci/get_bundle_requirements.py b/ci/get_bundle_requirements.py
index 71e69b1f..fdab36d3 100644
--- a/ci/get_bundle_requirements.py
+++ b/ci/get_bundle_requirements.py
@@ -53,7 +53,7 @@ def increment_version(version):
     return result_version
 
 
-def get_requirements(bundle, models_path):
+def get_requirements(bundle, models_path, requirements_file):
     """
     This function is used to produce a requirements txt file, and print a string
     which shows the filename. The printed string can be used in shell scripts.
@@ -86,14 +86,12 @@ def get_requirements(bundle, models_path):
                     else:
                         if "pytorch_version" in metadata.keys():
                             # remove torch from libs
-                            libs = [lib for lib in libs if "torch" not in lib]
+                            libs = [lib for lib in libs if "torch" not in lib or lib == "pytorch-ignite"]
 
         if len(libs) > 0:
-            requirements_file_name = f"requirements_{bundle}.txt"
-            with open(requirements_file_name, "w") as f:
+            with open(requirements_file, "w") as f:
                 for line in libs:
                     f.write(f"{line}\n")
-            print(requirements_file_name)
 
 
 def get_install_script(bundle):
@@ -109,6 +107,7 @@ def get_install_script(bundle):
     parser.add_argument("-b", "--b", type=str, help="bundle name.")
     parser.add_argument("-p", "--p", type=str, default="models", help="models path.")
     parser.add_argument("--get_script", type=bool, default=False, help="whether to get the install script.")
+    parser.add_argument("--requirements_file", type=str, help="output filename for requirements.")
     args = parser.parse_args()
     bundle = args.b
     models_path = args.p
@@ -116,4 +115,5 @@ def get_install_script(bundle):
     if get_script is True:
         get_install_script(bundle)
     else:
-        get_requirements(bundle, models_path)
+        requirements_file = args.requirements_file
+        get_requirements(bundle, models_path, requirements_file)
diff --git a/ci/run_premerge_cpu.sh b/ci/run_premerge_cpu.sh
index 5312d0a2..8e84ad21 100755
--- a/ci/run_premerge_cpu.sh
+++ b/ci/run_premerge_cpu.sh
@@ -49,6 +49,8 @@ verify_bundle() {
     done
     echo 'Run verify bundle...'
     pip install -r requirements.txt
+    # install extra dependencies for get changed bundle
+    pip install jsonschema gdown pyyaml
     head_ref=$(git rev-parse HEAD)
     git fetch origin dev $head_ref
     # achieve all changed files in 'models'
@@ -66,18 +68,19 @@ verify_bundle() {
                 if is_excluded "$bundle"; then
                     echo "skip '$bundle' cpu premerge tests."
                 else
-                    pip install -r requirements-dev.txt
                     # get required libraries according to the bundle's metadata file
-                    requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+                    requirements_file="requirements_$bundle.txt"
+                    python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
                     # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
                     if [ $ALLOW_MONAI_RC = true ]; then
                         include_pre_release="--pre"
                     else
                         include_pre_release=""
                     fi
-                    if [ ! -z "$requirements" ]; then
+                    # Check if the requirements file exists and is not empty
+                    if [ -s "$requirements_file" ]; then
                         echo "install required libraries for bundle: $bundle"
-                        pip install $include_pre_release -r "$requirements"
+                        pip install $include_pre_release -r "$requirements_file"
                     fi
                     # verify bundle
                     python $(pwd)/ci/verify_bundle.py -b "$bundle" -m "min"  # min tests on cpu
diff --git a/ci/run_premerge_gpu.sh b/ci/run_premerge_gpu.sh
index 5d907f36..ae60f0bd 100755
--- a/ci/run_premerge_gpu.sh
+++ b/ci/run_premerge_gpu.sh
@@ -31,15 +31,18 @@ fi
 
 init_venv() {
     if [ ! -d "model_zoo_venv" ]; then  # Check if the venv directory does not exist
-        echo "initializing pip environment: $1"
+        echo "initializing pip environment"
         python -m venv model_zoo_venv
         source model_zoo_venv/bin/activate
         pip install --upgrade pip wheel
-        pip install -r $1
+        pip install jsonschema gdown pyyaml parameterized fire
         export PYTHONPATH=$PWD
     else
         echo "Virtual environment model_zoo_venv already exists. Activating..."
         source model_zoo_venv/bin/activate
+        pip install --upgrade pip wheel
+        pip install jsonschema gdown pyyaml parameterized fire
+        export PYTHONPATH=$PWD
     fi
 }
 
@@ -54,9 +57,8 @@ remove_venv() {
 }
 
 set_local_env() {
-    echo "set local pip environment: $1"
+    echo "set local pip environment"
     pip install --upgrade pip wheel
-    pip install -r $1
     export PYTHONPATH=$PWD
 }
 
@@ -75,24 +77,20 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            init_venv
+            # Check if the requirements file exists and is not empty
+            if [ -s "$requirements_file" ]; then
                 echo "install required libraries for bundle: $bundle"
-                pip install $include_pre_release -r "$requirements"
+                pip install $include_pre_release -r "$requirements_file"
             fi
             # get extra install script if exists
             extra_script=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --get_script True)
diff --git a/ci/run_regular_tests_cpu.sh b/ci/run_regular_tests_cpu.sh
index 24900d58..183af8d8 100755
--- a/ci/run_regular_tests_cpu.sh
+++ b/ci/run_regular_tests_cpu.sh
@@ -34,21 +34,23 @@ verify_release_bundle() {
     echo 'Run verify bundle...'
     # get all bundles
     download_path="download"
-    pip install -r requirements-dev.txt
     pip install git+https://github.com/Project-MONAI/MONAI.git@dev  # project-monai/model-zoo issue #505
+    pip install jsonschema gdown
     # download bundle from releases
     python $(pwd)/ci/download_latest_bundle.py --b "$bundle" --models_path $(pwd)/models --p "$download_path"
     # get required libraries according to the bundle's metadata file
-    requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --p "$download_path")
+    requirements_file="requirements_$bundle.txt"
+    python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
     # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
     if [ $ALLOW_MONAI_RC = true ]; then
         include_pre_release="--pre"
     else
         include_pre_release=""
     fi
-    if [ ! -z "$requirements" ]; then
+    # Check if the requirements file exists and is not empty
+    if [ -s "$requirements_file" ]; then
         echo "install required libraries for bundle: $bundle"
-        pip install $include_pre_release -r "$requirements"
+        pip install $include_pre_release -r "$requirements_file"
     fi
     # verify bundle
     python $(pwd)/ci/verify_bundle.py -b "$bundle" -p "$download_path" -m "regular"  # regular tests on cpu
diff --git a/ci/utils.py b/ci/utils.py
index 49f2d823..3f26663b 100644
--- a/ci/utils.py
+++ b/ci/utils.py
@@ -17,11 +17,11 @@
 import shutil
 from typing import List
 
-from huggingface_hub import HfApi
 from monai.apps.utils import download_url
 from monai.bundle.config_parser import ConfigParser
 from monai.utils import look_up_option, optional_import
 
+huggingface_hub, _ = optional_import("huggingface_hub")
 Github, _ = optional_import("github", name="Github")
 
 SUPPORTED_HASH_TYPES = {"md5": hashlib.md5, "sha1": hashlib.sha1, "sha256": hashlib.sha256, "sha512": hashlib.sha512}
@@ -178,7 +178,7 @@ def get_existing_bundle_list(model_info):
 
 
 def create_bundle_to_huggingface(bundle_name: str, org_name: str):
-    api = HfApi()
+    api = huggingface_hub.HfApi()
     try:
         _ = api.create_repo(repo_id=f"{org_name}/{bundle_name}", repo_type="model", private=False)
     except Exception as e:
@@ -189,7 +189,7 @@ def create_bundle_to_huggingface(bundle_name: str, org_name: str):
 
 
 def upload_version_to_huggingface(bundle_name: str, version: str, root_path: str, org_name: str):
-    api = HfApi()
+    api = huggingface_hub.HfApi()
 
     try:
         # if no file is changed, will skip uploading automatically
diff --git a/models/brain_image_synthesis_latent_diffusion_model/configs/metadata.json b/models/brain_image_synthesis_latent_diffusion_model/configs/metadata.json
index a76b40db..35afed30 100644
--- a/models/brain_image_synthesis_latent_diffusion_model/configs/metadata.json
+++ b/models/brain_image_synthesis_latent_diffusion_model/configs/metadata.json
@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
-    "version": "1.0.1",
+    "version": "1.0.2",
     "changelog": {
+        "1.0.2": "fix missing dependencies",
         "1.0.1": "update to huggingface hosting",
         "1.0.0": "Initial release"
     },
@@ -9,7 +10,8 @@
     "pytorch_version": "2.5.1",
     "numpy_version": "1.26.4",
     "required_packages_version": {
-        "nibabel": "5.3.2"
+        "nibabel": "5.3.2",
+        "einops": "0.7.0"
     },
     "task": "Brain image synthesis",
     "description": "A generative model for creating high-resolution 3D brain MRI based on UK Biobank",
diff --git a/models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json b/models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json
index fe5ce1c1..577f99cc 100644
--- a/models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json
+++ b/models/brats_mri_axial_slices_generative_diffusion/configs/metadata.json
@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
-    "version": "1.1.2",
+    "version": "1.1.3",
     "changelog": {
+        "1.1.3": "update to huggingface hosting and fix missing dependencies",
         "1.1.2": "update issue for IgniteInfo",
         "1.1.1": "enable tensorrt",
         "1.1.0": "update to use monai 1.4, model ckpt not changed, rm GenerativeAI repo",
@@ -21,7 +22,10 @@
     "numpy_version": "1.24.4",
     "required_packages_version": {
         "nibabel": "5.2.1",
-        "lpips": "0.1.4"
+        "lpips": "0.1.4",
+        "tensorboard": "2.17.0",
+        "einops": "0.7.0",
+        "pytorch-ignite": "0.4.11"
     },
     "supported_apps": {},
     "name": "BraTS MRI axial slices latent diffusion generation",
diff --git a/models/brats_mri_generative_diffusion/configs/metadata.json b/models/brats_mri_generative_diffusion/configs/metadata.json
index ce2305ea..7d46cde6 100644
--- a/models/brats_mri_generative_diffusion/configs/metadata.json
+++ b/models/brats_mri_generative_diffusion/configs/metadata.json
@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
-    "version": "1.1.2",
+    "version": "1.1.3",
     "changelog": {
+        "1.1.3": "update to huggingface hosting and fix missing dependencies",
         "1.1.2": "update issue for IgniteInfo",
         "1.1.1": "enable tensorrt",
         "1.1.0": "update to use monai 1.4, model ckpt not changed, rm GenerativeAI repo",
@@ -21,7 +22,10 @@
     "numpy_version": "1.24.4",
     "required_packages_version": {
         "nibabel": "5.2.1",
-        "lpips": "0.1.4"
+        "lpips": "0.1.4",
+        "einops": "0.7.0",
+        "pytorch-ignite": "0.4.11",
+        "tensorboard": "2.17.0"
     },
     "supported_apps": {},
     "name": "BraTS MRI image latent diffusion generation",
diff --git a/models/brats_mri_segmentation/configs/metadata.json b/models/brats_mri_segmentation/configs/metadata.json
index c26134f9..93f01ac9 100644
--- a/models/brats_mri_segmentation/configs/metadata.json
+++ b/models/brats_mri_segmentation/configs/metadata.json
@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.5.2",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.5.3",
     "changelog": {
+        "0.5.3": "update to huggingface hosting",
         "0.5.2": "use monai 1.4 and update large files",
         "0.5.1": "update to use monai 1.3.1",
         "0.5.0": "add load_pretrain flag for infer",
@@ -33,12 +34,13 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "nibabel": "5.2.1",
         "pytorch-ignite": "0.4.11",
         "scikit-learn": "1.5.1",
         "tensorboard": "2.17.0"
     },
+    "supported_apps": {},
     "name": "BraTS MRI segmentation",
     "task": "Multimodal Brain Tumor segmentation",
     "description": "A pre-trained model for volumetric (3D) segmentation of brain tumor subregions from multimodal MRIs based on BraTS 2018 data",
diff --git a/models/breast_density_classification/configs/metadata.json b/models/breast_density_classification/configs/metadata.json
index 1cba67b3..f42e8f95 100644
--- a/models/breast_density_classification/configs/metadata.json
+++ b/models/breast_density_classification/configs/metadata.json
@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.1.6",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.1.7",
     "changelog": {
+        "0.1.7": "update to huggingface hosting",
         "0.1.6": "Remove meta dict usage",
         "0.1.5": "Fixed duplication of input output format section",
         "0.1.4": "Changed Readme",
@@ -13,9 +14,10 @@
     "monai_version": "1.3.0",
     "pytorch_version": "1.13.1",
     "numpy_version": "1.22.2",
-    "optional_packages_version": {
+    "required_packages_version": {
         "torchvision": "0.14.1"
     },
+    "supported_apps": {},
     "name": "Breast density classification",
     "task": "Breast Density Classification",
     "description": "A pre-trained model for classifying breast images (mammograms)  ",
diff --git a/models/classification_template/configs/metadata.json b/models/classification_template/configs/metadata.json
index 776901db..a37c2078 100644
--- a/models/classification_template/configs/metadata.json
+++ b/models/classification_template/configs/metadata.json
@@ -1,16 +1,19 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.0.2",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.0.3",
     "changelog": {
+        "0.0.3": "update to huggingface hosting",
         "0.0.2": "update large file yml",
         "0.0.1": "Initial version"
     },
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
-        "pytorch-ignite": "0.4.11"
+    "required_packages_version": {
+        "pytorch-ignite": "0.4.11",
+        "pyyaml": "6.0.2"
     },
+    "supported_apps": {},
     "name": "Classification Template",
     "task": "Classification Template in 2D images",
     "description": "This is a template bundle for classifying in 2D, take this as a basis for your own bundles.",
diff --git a/models/endoscopic_inbody_classification/configs/metadata.json b/models/endoscopic_inbody_classification/configs/metadata.json
index 7e0510e4..535fb8c6 100644
--- a/models/endoscopic_inbody_classification/configs/metadata.json
+++ b/models/endoscopic_inbody_classification/configs/metadata.json
@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.4.9",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.5.0",
     "changelog": {
+        "0.5.0": "update to huggingface hosting and fix missing dependencies",
         "0.4.9": "use monai 1.4 and update large files",
         "0.4.8": "update to use monai 1.3.1",
         "0.4.7": "add load_pretrain flag for infer",
@@ -31,10 +32,13 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "nibabel": "5.2.1",
-        "pytorch-ignite": "0.4.11"
+        "pytorch-ignite": "0.4.11",
+        "pillow": "10.4.0",
+        "tensorboard": "2.17.0"
     },
+    "supported_apps": {},
     "name": "Endoscopic inbody classification",
     "task": "Endoscopic inbody classification",
     "description": "A pre-trained binary classification model for endoscopic inbody classification task",
diff --git a/models/endoscopic_tool_segmentation/configs/metadata.json b/models/endoscopic_tool_segmentation/configs/metadata.json
index 2d72b886..ef84e9c1 100644
--- a/models/endoscopic_tool_segmentation/configs/metadata.json
+++ b/models/endoscopic_tool_segmentation/configs/metadata.json
@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.6.0",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.6.1",
     "changelog": {
+        "0.6.1": "update to huggingface hosting and fix missing dependencies",
         "0.6.0": "use monai 1.4 and update large files",
         "0.5.9": "update to use monai 1.3.1",
         "0.5.8": "add load_pretrain flag for infer",
@@ -34,10 +35,13 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "nibabel": "5.2.1",
-        "pytorch-ignite": "0.4.11"
+        "pytorch-ignite": "0.4.11",
+        "pillow": "10.4.0",
+        "tensorboard": "2.17.0"
     },
+    "supported_apps": {},
     "name": "Endoscopic tool segmentation",
     "task": "Endoscopic tool segmentation",
     "description": "A pre-trained binary segmentation model for endoscopic tool segmentation",
diff --git a/models/lung_nodule_ct_detection/configs/metadata.json b/models/lung_nodule_ct_detection/configs/metadata.json
index 4bc75d27..999c4744 100644
--- a/models/lung_nodule_ct_detection/configs/metadata.json
+++ b/models/lung_nodule_ct_detection/configs/metadata.json
@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.6.8",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.6.9",
     "changelog": {
+        "0.6.9": "update to huggingface hosting and fix missing dependencies",
         "0.6.8": "update issue for IgniteInfo",
         "0.6.7": "use monai 1.4 and update large files",
         "0.6.6": "update to use monai 1.3.1",
@@ -35,11 +36,13 @@
     "monai_version": "1.4.0",
     "pytorch_version": "2.4.0",
     "numpy_version": "1.24.4",
-    "optional_packages_version": {
+    "required_packages_version": {
         "nibabel": "5.2.1",
         "pytorch-ignite": "0.4.11",
-        "torchvision": "0.19.0"
+        "torchvision": "0.19.0",
+        "tensorboard": "2.17.0"
     },
+    "supported_apps": {},
     "name": "Lung nodule CT detection",
     "task": "CT lung nodule detection",
     "description": "A pre-trained model for volumetric (3D) detection of the lung lesion from CT image on LUNA16 dataset",
diff --git a/models/maisi_ct_generative/configs/metadata.json b/models/maisi_ct_generative/configs/metadata.json
index f46c2307..e40677d5 100644
--- a/models/maisi_ct_generative/configs/metadata.json
+++ b/models/maisi_ct_generative/configs/metadata.json
@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_ldm_20240318.json",
-    "version": "1.0.0",
+    "version": "1.0.1",
     "changelog": {
+        "1.0.1": "add missing dependencies",
         "1.0.0": "accelerated maisi, inference only, is not compartible with previous maisi diffusion model weights",
         "0.4.6": "add TensorRT support",
         "0.4.5": "update README",
@@ -18,7 +19,12 @@
     "optional_packages_version": {
         "fire": "0.6.0",
         "nibabel": "5.2.1",
-        "tqdm": "4.66.4"
+        "tqdm": "4.66.4",
+        "einops": "0.7.0",
+        "scikit-image": "0.23.2",
+        "pytorch-ignite": "0.4.11",
+        "tensorboard": "2.17.0",
+        "itk": "5.4.0"
     },
     "supported_apps": {
         "maisi-nim": ""
diff --git a/models/mednist_ddpm/configs/metadata.json b/models/mednist_ddpm/configs/metadata.json
index 0d5294a7..24a03c25 100644
--- a/models/mednist_ddpm/configs/metadata.json
+++ b/models/mednist_ddpm/configs/metadata.json
@@ -1,14 +1,17 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
-    "version": "1.0.1",
+    "version": "1.0.2",
     "changelog": {
+        "1.0.2": "add missing dependencies",
         "1.0.1": "update to huggingface hosting",
         "1.0.0": "Initial release"
     },
     "monai_version": "1.4.0",
     "pytorch_version": "2.5.1",
     "numpy_version": "1.26.4",
-    "required_packages_version": {},
+    "required_packages_version": {
+        "pyyaml": "6.0.2"
+    },
     "task": "MedNIST Hand Generation",
     "description": "",
     "authors": "Walter Hugo Lopez Pinaya, Mark Graham, and Eric Kerfoot",
diff --git a/models/mednist_gan/configs/metadata.json b/models/mednist_gan/configs/metadata.json
index c013d46c..bb7211c4 100644
--- a/models/mednist_gan/configs/metadata.json
+++ b/models/mednist_gan/configs/metadata.json
@@ -1,7 +1,8 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_20220718.json",
-    "version": "0.4.2",
+    "version": "0.4.3",
     "changelog": {
+        "0.4.3": "update to huggingface hosting and compatible with py3.10",
         "0.4.2": "add name tag",
         "0.4.1": "fix license Copyright error",
         "0.4.0": "update license files",
@@ -9,12 +10,12 @@
         "0.2.0": "Unify naming",
         "0.1.0": "Initial version"
     },
-    "monai_version": "1.0.0",
-    "pytorch_version": "1.10.0",
-    "numpy_version": "1.21.0",
+    "monai_version": "1.4.0",
+    "pytorch_version": "2.4.0",
+    "numpy_version": "1.24.4",
     "optional_packages_version": {
-        "pytorch-ignite": "0.4.8",
-        "pillow": "8.4.0"
+        "pytorch-ignite": "0.4.11",
+        "pillow": "10.4.0"
     },
     "name": "MedNIST GAN",
     "task": "Generate random hand images from the MedNIST dataset",
diff --git a/models/mednist_reg/configs/metadata.json b/models/mednist_reg/configs/metadata.json
index 53c052b5..8615a69c 100644
--- a/models/mednist_reg/configs/metadata.json
+++ b/models/mednist_reg/configs/metadata.json
@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.0.5",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.0.6",
     "changelog": {
+        "0.0.6": "update to huggingface hosting",
         "0.0.5": "update large files",
         "0.0.4": "add name tag",
         "0.0.3": "update to use monai 1.1.0",
@@ -11,9 +12,11 @@
     "monai_version": "1.1.0",
     "pytorch_version": "1.13.0",
     "numpy_version": "1.22.2",
-    "optional_packages_version": {
-        "pytorch-ignite": "0.4.8"
+    "required_packages_version": {
+        "pytorch-ignite": "0.4.8",
+        "pyyaml": "6.0.2"
     },
+    "supported_apps": {},
     "name": "MedNIST registration",
     "task": "Spatial transformer for hand image registration from the MedNIST dataset",
     "description": "This is an example of a ResNet and spatial transformer for hand xray image registration",
diff --git a/models/multi_organ_segmentation/configs/metadata.json b/models/multi_organ_segmentation/configs/metadata.json
index d7ba2f47..0b1fa7e2 100644
--- a/models/multi_organ_segmentation/configs/metadata.json
+++ b/models/multi_organ_segmentation/configs/metadata.json
@@ -1,7 +1,8 @@
 {
-    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json",
-    "version": "0.0.4",
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.0.5",
     "changelog": {
+        "0.0.5": "update to huggingface hosting",
         "0.0.4": "Set image_only to False",
         "0.0.3": "Update for stable MONAI version",
         "0.0.2": "Retrain with new MONAI",
@@ -10,11 +11,13 @@
     "monai_version": "1.3.0",
     "pytorch_version": "1.13.1",
     "numpy_version": "1.22.2",
-    "optional_packages_version": {
+    "required_packages_version": {
         "fire": "0.5.0",
         "nibabel": "5.1.0",
-        "pytorch-ignite": "0.4.11"
+        "pytorch-ignite": "0.4.11",
+        "pyyaml": "6.0.2"
     },
+    "supported_apps": {},
     "name": "Abdominal multi-organ segmentation",
     "task": "Multi-organ segmentation in abdominal CT",
     "description": "DiNTS architectures for volumetric (3D) segmentation of the abdominal from CT image",
diff --git a/requirements.txt b/requirements.txt
index 6d29a604..8ba701d3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1 @@
 monai>=1.0.1
-huggingface_hub==0.29.3
-jsonschema


Reviews:
user: login: yiheng-wang-nv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:07:52Z
user: login: yiheng-wang-nv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-24T09:08:45Z
user: login: yiheng-wang-nv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T08:54:10Z
user: login: Nic-Ma
body: Hi @KumoLiu , could you please help review this PR?

Thanks in advance.

state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T14:40:52Z
user: login: yiheng-wang-nv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T15:03:40Z
user: login: Nic-Ma
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-25T22:45:15Z
user: login: yiheng-wang-nv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-26T02:10:02Z
user: login: mingxin-zheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-26T04:26:58Z
user: login: yiheng-wang-nv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-26T05:20:05Z
user: login: yiheng-wang-nv
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-26T06:45:48Z
user: login: Nic-Ma
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-26T08:12:16Z
user: login: mingxin-zheng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-26T08:14:26Z

Review comments:
diff_hunk: @@ -14,23 +14,14 @@
 import os
 
 from monai.bundle import download
-from utils import get_checksum, get_hash_func, get_latest_version, get_version_checksum
+from utils import get_latest_version
path: ci/download_latest_bundle.py
user: login: yiheng-wang-nv
body: For huggingface hosting, each bundle's each version has it's commit, no need to use checksum to ensure consistency. cc @ericspod @KumoLiu 
created_at: 2025-03-24T09:07:52Z
author_association: COLLABORATOR
diff_hunk: @@ -1,20 +1,21 @@
 {
     "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_20220718.json",
-    "version": "0.4.2",
+    "version": "0.4.3",
     "changelog": {
+        "0.4.3": "update to huggingface hosting and compatible with py3.10",
         "0.4.2": "add name tag",
         "0.4.1": "fix license Copyright error",
         "0.4.0": "update license files",
         "0.3.0": "Update for 1.0",
         "0.2.0": "Unify naming",
         "0.1.0": "Initial version"
     },
-    "monai_version": "1.0.0",
-    "pytorch_version": "1.10.0",
-    "numpy_version": "1.21.0",
+    "monai_version": "1.4.0",
path: models/mednist_gan/configs/metadata.json
user: login: yiheng-wang-nv
body: python3.10 (the version that our CI is used) does not support torch 1.10, thus I updated the dependencies for this bundle. cc @ericspod @KumoLiu 
created_at: 2025-03-24T09:08:45Z
author_association: COLLABORATOR
diff_hunk: @@ -75,24 +77,27 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
path: ci/run_premerge_gpu.sh
user: login: yiheng-wang-nv
body: we should ensure the test env is clean for each bundle, otherwise CI cannot find missing dependencies and make have lib conflicts
created_at: 2025-03-25T08:54:10Z
author_association: COLLABORATOR
diff_hunk: @@ -75,24 +77,27 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            # Set a new environment to verify bundle
+            if [ "$bundle" == "maisi_ct_generative" ]; then
path: ci/run_premerge_gpu.sh
user: login: Nic-Ma
body: Why make MAISI specific logic?

Thanks.

created_at: 2025-03-25T14:40:10Z
author_association: COLLABORATOR
diff_hunk: @@ -75,24 +77,27 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            # Set a new environment to verify bundle
+            if [ "$bundle" == "maisi_ct_generative" ]; then
path: ci/run_premerge_gpu.sh
user: login: yiheng-wang-nv
body: Hi @Nic-Ma , as mentioned in the line below, we use venv to setup environment for each bundle, but maisi requires `xformers` which cannot work with it
created_at: 2025-03-25T15:03:40Z
author_association: COLLABORATOR
diff_hunk: @@ -75,24 +77,27 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            # Set a new environment to verify bundle
+            if [ "$bundle" == "maisi_ct_generative" ]; then
path: ci/run_premerge_gpu.sh
user: login: Nic-Ma
body: Hi @mingxin-zheng , I remember you know something about the `xformers` installation? Could you please help review this PR?

Thanks.

created_at: 2025-03-25T22:45:15Z
author_association: COLLABORATOR
diff_hunk: @@ -75,24 +77,27 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            # Set a new environment to verify bundle
+            if [ "$bundle" == "maisi_ct_generative" ]; then
path: ci/run_premerge_gpu.sh
user: login: yiheng-wang-nv
body: The code change does not impact the maisi's logic, and I only adjust the order to ensure:
1) the container environment is used to get requirements for each bundle
2) a new venv is used to verify each bundle
cc @Nic-Ma @mingxin-zheng 
created_at: 2025-03-26T02:10:02Z
author_association: COLLABORATOR
diff_hunk: @@ -75,24 +77,27 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            # Set a new environment to verify bundle
+            if [ "$bundle" == "maisi_ct_generative" ]; then
path: ci/run_premerge_gpu.sh
user: login: mingxin-zheng
body: I remember MAISI doesn't require `xformers` anymore in version before the latest release, but I could be wrong.
created_at: 2025-03-26T04:26:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -75,24 +77,27 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            # Set a new environment to verify bundle
+            if [ "$bundle" == "maisi_ct_generative" ]; then
path: ci/run_premerge_gpu.sh
user: login: yiheng-wang-nv
body: Thanks for the info @mingxin-zheng . The current method is implemented when we involved the initial MAISI, maybe now `xformers` is not necessary. Let me check it
created_at: 2025-03-26T05:20:05Z
author_association: COLLABORATOR
diff_hunk: @@ -75,24 +77,27 @@ verify_bundle() {
             python $(pwd)/ci/prepare_schema.py --l "$bundle_list"
         for bundle in $bundle_list;
         do
-            # Check if the bundle is "maisi_ct_generative", if so, set local environment (venv cannot work with xformers)
-            if [ "$bundle" == "maisi_ct_generative" ]; then
-                echo "Special handling for maisi_ct_generative bundle"
-                set_local_env requirements-dev.txt
-            else
-                init_venv requirements-dev.txt
-            fi
             # get required libraries according to the bundle's metadata file
-            requirements=$(python $(pwd)/ci/get_bundle_requirements.py --b "$bundle")
+            requirements_file="requirements_$bundle.txt"
+            python $(pwd)/ci/get_bundle_requirements.py --b "$bundle" --requirements_file "$requirements_file"
             # check if ALLOW_MONAI_RC is set to 1, if so, append --pre to the pip install command
             if [ $ALLOW_MONAI_RC = true ]; then
                 include_pre_release="--pre"
             else
                 include_pre_release=""
             fi
-            if [ ! -z "$requirements" ]; then
+            # Set a new environment to verify bundle
+            if [ "$bundle" == "maisi_ct_generative" ]; then
path: ci/run_premerge_gpu.sh
user: login: yiheng-wang-nv
body: updated
created_at: 2025-03-26T06:45:48Z
author_association: COLLABORATOR
diff_hunk: @@ -178,7 +178,7 @@ def get_existing_bundle_list(model_info):
 
 
 def create_bundle_to_huggingface(bundle_name: str, org_name: str):
path: ci/utils.py
user: login: mingxin-zheng
body: Not a blocker, but I personally think such tool can be included in the monai.bundle, so that the user can publish their bundle to HF easily.
created_at: 2025-03-26T08:14:26Z
author_association: CONTRIBUTOR

PR comments:
user: login: yiheng-wang-nv
created_at: 2025-03-24T08:49:06Z
body: /build
author_association: COLLABORATOR
user: login: yiheng-wang-nv
created_at: 2025-03-24T09:18:01Z
body: /build
author_association: COLLABORATOR
user: login: yiheng-wang-nv
created_at: 2025-03-25T03:44:13Z
body: /build
author_association: COLLABORATOR
user: login: yiheng-wang-nv
created_at: 2025-03-25T06:27:44Z
body: /build
author_association: COLLABORATOR
user: login: yiheng-wang-nv
created_at: 2025-03-25T07:03:07Z
body: /build
author_association: COLLABORATOR
user: login: yiheng-wang-nv
created_at: 2025-03-25T08:54:41Z
body: /build
author_association: COLLABORATOR
user: login: yiheng-wang-nv
created_at: 2025-03-25T09:17:57Z
body: /build
author_association: COLLABORATOR
user: login: yiheng-wang-nv
created_at: 2025-03-25T09:37:26Z
body: /build
author_association: COLLABORATOR
user: login: yiheng-wang-nv
created_at: 2025-03-26T06:45:59Z
body: /build
author_association: COLLABORATOR