untroubled files first

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.o filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+__pycache__/

README.md

@@ -1,6 +1,6 @@
 ---
 title: LucidDreamer
-emoji: 👁
+emoji: 😴
 colorFrom: purple
 colorTo: pink
 sdk: gradio
@@ -11,3 +11,5 @@ license: cc-by-nc-sa-4.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+More information in https://luciddreamer-cvlab.github.io

ZoeDepth/.gitignore

@@ -0,0 +1,148 @@
+*.png
+**.gif
+.vscode/
+*.rdb
+**.xml
+wandb/
+slurm/
+tmp/
+.logs/
+checkpoints/
+external_jobs/
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+ptlflow_logs/
+output/
+log/
+.idea/
+# C extensions
+*.so
+results/
+**.DS_Store
+**.pt
+demo/
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+~shortcuts/
+**/wandb_logs/
+**.db
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

ZoeDepth/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Intelligent Systems Lab Org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ZoeDepth/README.md

@@ -0,0 +1,248 @@
+# **ZoeDepth: Combining relative and metric depth** (Official implementation)  <!-- omit in toc -->
+[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/isl-org/ZoeDepth)
+[![Open in Spaces](https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg)](https://huggingface.co/spaces/shariqfarooq/ZoeDepth)
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) ![PyTorch](https://img.shields.io/badge/PyTorch_v1.10.1-EE4C2C?&logo=pytorch&logoColor=white) 
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/zoedepth-zero-shot-transfer-by-combining/monocular-depth-estimation-on-nyu-depth-v2)](https://paperswithcode.com/sota/monocular-depth-estimation-on-nyu-depth-v2?p=zoedepth-zero-shot-transfer-by-combining)
+
+>#### [ZoeDepth: Zero-shot Transfer by Combining Relative and Metric Depth](https://arxiv.org/abs/2302.12288)
+> ##### [Shariq Farooq Bhat](https://shariqfarooq123.github.io), [Reiner Birkl](https://www.researchgate.net/profile/Reiner-Birkl), [Diana Wofk](https://dwofk.github.io/), [Peter Wonka](http://peterwonka.net/), [Matthias Müller](https://matthias.pw/)
+
+[[Paper]](https://arxiv.org/abs/2302.12288)
+
+![teaser](assets/zoedepth-teaser.png)
+
+## **Table of Contents** <!-- omit in toc -->
+- [**Usage**](#usage)
+  - [Using torch hub](#using-torch-hub)
+  - [Using local copy](#using-local-copy)
+    - [Using local torch hub](#using-local-torch-hub)
+    - [or load the models manually](#or-load-the-models-manually)
+  - [Using ZoeD models to predict depth](#using-zoed-models-to-predict-depth)
+- [**Environment setup**](#environment-setup)
+- [**Sanity checks** (Recommended)](#sanity-checks-recommended)
+- [Model files](#model-files)
+- [**Evaluation**](#evaluation)
+  - [Evaluating offical models](#evaluating-offical-models)
+  - [Evaluating local checkpoint](#evaluating-local-checkpoint)
+- [**Training**](#training)
+- [**Gradio demo**](#gradio-demo)
+- [**Citation**](#citation)
+
+
+## **Usage**
+It is recommended to fetch the latest [MiDaS repo](https://github.com/isl-org/MiDaS) via torch hub before proceeding:
+```python
+import torch
+
+torch.hub.help("intel-isl/MiDaS", "DPT_BEiT_L_384", force_reload=True)  # Triggers fresh download of MiDaS repo
+```
+### **ZoeDepth models** <!-- omit in toc -->
+### Using torch hub
+```python
+import torch
+
+repo = "isl-org/ZoeDepth"
+# Zoe_N
+model_zoe_n = torch.hub.load(repo, "ZoeD_N", pretrained=True)
+
+# Zoe_K
+model_zoe_k = torch.hub.load(repo, "ZoeD_K", pretrained=True)
+
+# Zoe_NK
+model_zoe_nk = torch.hub.load(repo, "ZoeD_NK", pretrained=True)
+```
+### Using local copy
+Clone this repo:
+```bash
+git clone https://github.com/isl-org/ZoeDepth.git && cd ZoeDepth
+```
+#### Using local torch hub
+You can use local source for torch hub to load the ZoeDepth models, for example: 
+```python
+import torch
+
+# Zoe_N
+model_zoe_n = torch.hub.load(".", "ZoeD_N", source="local", pretrained=True)
+```
+
+#### or load the models manually
+```python
+from zoedepth.models.builder import build_model
+from zoedepth.utils.config import get_config
+
+# ZoeD_N
+conf = get_config("zoedepth", "infer")
+model_zoe_n = build_model(conf)
+
+# ZoeD_K
+conf = get_config("zoedepth", "infer", config_version="kitti")
+model_zoe_k = build_model(conf)
+
+# ZoeD_NK
+conf = get_config("zoedepth_nk", "infer")
+model_zoe_nk = build_model(conf)
+```
+
+### Using ZoeD models to predict depth 
+```python
+##### sample prediction
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+zoe = model_zoe_n.to(DEVICE)
+
+
+# Local file
+from PIL import Image
+image = Image.open("/path/to/image.jpg").convert("RGB")  # load
+depth_numpy = zoe.infer_pil(image)  # as numpy
+
+depth_pil = zoe.infer_pil(image, output_type="pil")  # as 16-bit PIL Image
+
+depth_tensor = zoe.infer_pil(image, output_type="tensor")  # as torch tensor
+
+
+
+# Tensor 
+from zoedepth.utils.misc import pil_to_batched_tensor
+X = pil_to_batched_tensor(image).to(DEVICE)
+depth_tensor = zoe.infer(X)
+
+
+
+# From URL
+from zoedepth.utils.misc import get_image_from_url
+
+# Example URL
+URL = "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS4W8H_Nxk_rs3Vje_zj6mglPOH7bnPhQitBH8WkqjlqQVotdtDEG37BsnGofME3_u6lDk&usqp=CAU"
+
+
+image = get_image_from_url(URL)  # fetch
+depth = zoe.infer_pil(image)
+
+# Save raw
+from zoedepth.utils.misc import save_raw_16bit
+fpath = "/path/to/output.png"
+save_raw_16bit(depth, fpath)
+
+# Colorize output
+from zoedepth.utils.misc import colorize
+
+colored = colorize(depth)
+
+# save colored output
+fpath_colored = "/path/to/output_colored.png"
+Image.fromarray(colored).save(fpath_colored)
+```
+
+## **Environment setup**
+The project depends on :
+- [pytorch](https://pytorch.org/) (Main framework)
+- [timm](https://timm.fast.ai/)  (Backbone helper for MiDaS)
+- pillow, matplotlib, scipy, h5py, opencv (utilities)
+
+Install environment using `environment.yml` : 
+
+Using [mamba](https://github.com/mamba-org/mamba) (fastest):
+```bash
+mamba env create -n zoe --file environment.yml
+mamba activate zoe
+```
+Using conda : 
+
+```bash
+conda env create -n zoe --file environment.yml
+conda activate zoe
+```
+
+## **Sanity checks** (Recommended)
+Check if models can be loaded: 
+```bash
+python sanity_hub.py
+```
+Try a demo prediction pipeline:
+```bash
+python sanity.py
+```
+This will save a file `pred.png` in the root folder, showing RGB and corresponding predicted depth side-by-side.
+## Model files
+Models are defined under `models/` folder, with `models/<model_name>_<version>.py` containing model definitions and  `models/config_<model_name>.json` containing configuration.
+
+Single metric head models (Zoe_N and Zoe_K from the paper) have the common definition and are defined under `models/zoedepth` while as the multi-headed model (Zoe_NK) is defined under `models/zoedepth_nk`.
+## **Evaluation**
+Download the required dataset and change the `DATASETS_CONFIG` dictionary in `utils/config.py` accordingly. 
+### Evaluating offical models
+On NYU-Depth-v2 for example:
+
+For ZoeD_N:
+```bash
+python evaluate.py -m zoedepth -d nyu
+```
+
+For ZoeD_NK:
+```bash
+python evaluate.py -m zoedepth_nk -d nyu
+```
+
+### Evaluating local checkpoint
+```bash
+python evaluate.py -m zoedepth --pretrained_resource="local::/path/to/local/ckpt.pt" -d nyu
+```
+Pretrained resources are prefixed with `url::` to indicate weights should be fetched from a url, or `local::` to indicate path is a local file. Refer to `models/model_io.py` for details. 
+
+The dataset name should match the corresponding key in `utils.config.DATASETS_CONFIG` .
+
+## **Training**
+Download training datasets as per instructions given [here](https://github.com/cleinc/bts/tree/master/pytorch#nyu-depvh-v2). Then for training a single head model on NYU-Depth-v2 :
+```bash
+python train_mono.py -m zoedepth --pretrained_resource=""
+```
+
+For training the Zoe-NK model:
+```bash
+python train_mix.py -m zoedepth_nk --pretrained_resource=""
+```
+## **Gradio demo**
+We provide a UI demo built using [gradio](https://gradio.app/). To get started, install UI requirements:
+```bash
+pip install -r ui/ui_requirements.txt
+```
+Then launch the gradio UI:
+```bash
+python -m ui.app
+```
+
+The UI is also hosted on HuggingFace🤗 [here](https://huggingface.co/spaces/shariqfarooq/ZoeDepth)
+## **Citation**
+```
+@misc{https://doi.org/10.48550/arxiv.2302.12288,
+  doi = {10.48550/ARXIV.2302.12288},
+  
+  url = {https://arxiv.org/abs/2302.12288},
+  
+  author = {Bhat, Shariq Farooq and Birkl, Reiner and Wofk, Diana and Wonka, Peter and Müller, Matthias},
+  
+  keywords = {Computer Vision and Pattern Recognition (cs.CV), FOS: Computer and information sciences, FOS: Computer and information sciences},
+  
+  title = {ZoeDepth: Zero-shot Transfer by Combining Relative and Metric Depth},
+  
+  publisher = {arXiv},
+  
+  year = {2023},
+  
+  copyright = {arXiv.org perpetual, non-exclusive license}
+}
+
+```
+
+
+
+
+
+
+
+
+
+
+
+
+

ZoeDepth/environment.yml

@@ -0,0 +1,26 @@
+name: zoe
+channels:
+  - pytorch
+  - nvidia
+  - conda-forge
+dependencies:
+  - cuda=11.7.1
+  - h5py=3.7.0
+  - hdf5=1.12.2
+  - matplotlib=3.6.2
+  - matplotlib-base=3.6.2
+  - numpy=1.24.1
+  - opencv=4.6.0
+  - pip=22.3.1
+  - python=3.9.7
+  - pytorch=1.13.1
+  - pytorch-cuda=11.7
+  - pytorch-mutex=1.0
+  - scipy=1.10.0
+  - torchaudio=0.13.1
+  - torchvision=0.14.1
+  - pip:
+    - huggingface-hub==0.11.1
+    - timm==0.6.12
+    - tqdm==4.64.1
+    - wandb==0.13.9

ZoeDepth/evaluate.py

@@ -0,0 +1,160 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import argparse
+from pprint import pprint
+
+import torch
+from zoedepth.utils.easydict import EasyDict as edict
+from tqdm import tqdm
+
+from zoedepth.data.data_mono import DepthDataLoader
+from zoedepth.models.builder import build_model
+from zoedepth.utils.arg_utils import parse_unknown
+from zoedepth.utils.config import change_dataset, get_config, ALL_EVAL_DATASETS, ALL_INDOOR, ALL_OUTDOOR
+from zoedepth.utils.misc import (RunningAverageDict, colors, compute_metrics,
+                        count_parameters)
+
+
+@torch.no_grad()
+def infer(model, images, **kwargs):
+    """Inference with flip augmentation"""
+    # images.shape = N, C, H, W
+    def get_depth_from_prediction(pred):
+        if isinstance(pred, torch.Tensor):
+            pred = pred  # pass
+        elif isinstance(pred, (list, tuple)):
+            pred = pred[-1]
+        elif isinstance(pred, dict):
+            pred = pred['metric_depth'] if 'metric_depth' in pred else pred['out']
+        else:
+            raise NotImplementedError(f"Unknown output type {type(pred)}")
+        return pred
+
+    pred1 = model(images, **kwargs)
+    pred1 = get_depth_from_prediction(pred1)
+
+    pred2 = model(torch.flip(images, [3]), **kwargs)
+    pred2 = get_depth_from_prediction(pred2)
+    pred2 = torch.flip(pred2, [3])
+
+    mean_pred = 0.5 * (pred1 + pred2)
+
+    return mean_pred
+
+
+@torch.no_grad()
+def evaluate(model, test_loader, config, round_vals=True, round_precision=3):
+    model.eval()
+    metrics = RunningAverageDict()
+    for i, sample in tqdm(enumerate(test_loader), total=len(test_loader)):
+        if 'has_valid_depth' in sample:
+            if not sample['has_valid_depth']:
+                continue
+        image, depth = sample['image'], sample['depth']
+        image, depth = image.cuda(), depth.cuda()
+        depth = depth.squeeze().unsqueeze(0).unsqueeze(0)
+        focal = sample.get('focal', torch.Tensor(
+            [715.0873]).cuda())  # This magic number (focal) is only used for evaluating BTS model
+        pred = infer(model, image, dataset=sample['dataset'][0], focal=focal)
+
+        # Save image, depth, pred for visualization
+        if "save_images" in config and config.save_images:
+            import os
+            # print("Saving images ...")
+            from PIL import Image
+            import torchvision.transforms as transforms
+            from zoedepth.utils.misc import colorize
+
+            os.makedirs(config.save_images, exist_ok=True)
+            # def save_image(img, path):
+            d = colorize(depth.squeeze().cpu().numpy(), 0, 10)
+            p = colorize(pred.squeeze().cpu().numpy(), 0, 10)
+            im = transforms.ToPILImage()(image.squeeze().cpu())
+            im.save(os.path.join(config.save_images, f"{i}_img.png"))
+            Image.fromarray(d).save(os.path.join(config.save_images, f"{i}_depth.png"))
+            Image.fromarray(p).save(os.path.join(config.save_images, f"{i}_pred.png"))
+
+
+
+        # print(depth.shape, pred.shape)
+        metrics.update(compute_metrics(depth, pred, config=config))
+
+    if round_vals:
+        def r(m): return round(m, round_precision)
+    else:
+        def r(m): return m
+    metrics = {k: r(v) for k, v in metrics.get_value().items()}
+    return metrics
+
+def main(config):
+    model = build_model(config)
+    test_loader = DepthDataLoader(config, 'online_eval').data
+    model = model.cuda()
+    metrics = evaluate(model, test_loader, config)
+    print(f"{colors.fg.green}")
+    print(metrics)
+    print(f"{colors.reset}")
+    metrics['#params'] = f"{round(count_parameters(model, include_all=True)/1e6, 2)}M"
+    return metrics
+
+
+def eval_model(model_name, pretrained_resource, dataset='nyu', **kwargs):
+
+    # Load default pretrained resource defined in config if not set
+    overwrite = {**kwargs, "pretrained_resource": pretrained_resource} if pretrained_resource else kwargs
+    config = get_config(model_name, "eval", dataset, **overwrite)
+    # config = change_dataset(config, dataset)  # change the dataset
+    pprint(config)
+    print(f"Evaluating {model_name} on {dataset}...")
+    metrics = main(config)
+    return metrics
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-m", "--model", type=str,
+                        required=True, help="Name of the model to evaluate")
+    parser.add_argument("-p", "--pretrained_resource", type=str,
+                        required=False, default=None, help="Pretrained resource to use for fetching weights. If not set, default resource from model config is used,  Refer models.model_io.load_state_from_resource for more details.")
+    parser.add_argument("-d", "--dataset", type=str, required=False,
+                        default='nyu', help="Dataset to evaluate on")
+
+    args, unknown_args = parser.parse_known_args()
+    overwrite_kwargs = parse_unknown(unknown_args)
+
+    if "ALL_INDOOR" in args.dataset:
+        datasets = ALL_INDOOR
+    elif "ALL_OUTDOOR" in args.dataset:
+        datasets = ALL_OUTDOOR
+    elif "ALL" in args.dataset:
+        datasets = ALL_EVAL_DATASETS
+    elif "," in args.dataset:
+        datasets = args.dataset.split(",")
+    else:
+        datasets = [args.dataset]
+    
+    for dataset in datasets:
+        eval_model(args.model, pretrained_resource=args.pretrained_resource,
+                    dataset=dataset, **overwrite_kwargs)

ZoeDepth/hubconf.py

@@ -0,0 +1,154 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+dependencies=['torch']
+from zoedepth.utils.config import get_config
+from zoedepth.models.builder import build_model
+import numpy as np
+import torch
+
+
+# ZoeD_N
+def ZoeD_N(pretrained=False, midas_model_type="DPT_BEiT_L_384", config_mode="infer", **kwargs):
+    """Zoe_M12_N model. This is the version of ZoeDepth that has a single metric head
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on NYU-Depth-V2
+        midas_model_type (str): Midas model type. Should be one of the models as listed in torch.hub.list("intel-isl/MiDaS"). Default: DPT_BEiT_L_384
+        config_mode (str): Config mode. Should be one of "infer", "train" or "eval". Default: "infer"
+    
+    Keyword Args:
+        **kwargs: Additional arguments to pass to the model
+        The following arguments are supported:
+            train_midas (bool): If True, returns a model that with trainable midas base. Default: False
+            use_pretrained_midas (bool): If True, returns a model that uses pretrained midas base. Default: False
+            n_bins (int): Number of bin centers. Defaults to 64.
+            bin_centers_type (str): "normed" or "softplus". Activation type used for bin centers. For "normed" bin centers, linear normalization trick is applied. This results in bounded bin centers.
+                                               For "softplus", softplus activation is used and thus are unbounded. Defaults to "softplus".
+            bin_embedding_dim (int): bin embedding dimension. Defaults to 128.
+            min_depth (float): Lower bound for normed bin centers. Defaults to 1e-3.
+            max_depth (float): Upper bound for normed bin centers. Defaults to 10.
+            n_attractors (List[int]): Number of bin attractors at decoder layers. Defaults to [16, 8, 4, 1].
+            attractor_alpha (int): Proportional attractor strength. Refer to models.layers.attractor for more details. Defaults to 1000.
+            attractor_gamma (int): Exponential attractor strength. Refer to models.layers.attractor for more details. Defaults to 2.
+            attractor_kind (str): Attraction aggregation "sum" or "mean". Defaults to 'mean'.
+            attractor_type (str): Type of attractor to use; "inv" (Inverse attractor) or "exp" (Exponential attractor). Defaults to 'inv'.
+            min_temp (int): Lower bound for temperature of output probability distribution. Defaults to 0.0212.
+            max_temp (int): Upper bound for temperature of output probability distribution. Defaults to 50.
+            force_keep_ar (bool): If True, the model will keep the aspect ratio of the input image. Defaults to True.
+    """
+    if pretrained and midas_model_type != "DPT_BEiT_L_384":
+        raise ValueError(f"Only DPT_BEiT_L_384 MiDaS model is supported for pretrained Zoe_N model, got: {midas_model_type}")
+
+    if not pretrained:
+        pretrained_resource = None
+    else:
+        pretrained_resource = "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_N.pt"
+
+    config = get_config("zoedepth", config_mode, pretrained_resource=pretrained_resource, **kwargs)
+    model = build_model(config)
+    return model
+
+# ZoeD_K
+def ZoeD_K(pretrained=False, midas_model_type="DPT_BEiT_L_384", config_mode="infer", **kwargs):
+    """Zoe_M12_K model. This is the version of ZoeDepth that has a single metric head
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on NYU-Depth-V2
+        midas_model_type (str): Midas model type. Should be one of the models as listed in torch.hub.list("intel-isl/MiDaS"). Default: DPT_BEiT_L_384
+        config_mode (str): Config mode. Should be one of "infer", "train" or "eval". Default: "infer"
+    
+    Keyword Args:
+        **kwargs: Additional arguments to pass to the model
+        The following arguments are supported:
+            train_midas (bool): If True, returns a model that with trainable midas base. Default: False
+            use_pretrained_midas (bool): If True, returns a model that uses pretrained midas base. Default: False
+            n_bins (int): Number of bin centers. Defaults to 64.
+            bin_centers_type (str): "normed" or "softplus". Activation type used for bin centers. For "normed" bin centers, linear normalization trick is applied. This results in bounded bin centers.
+                                               For "softplus", softplus activation is used and thus are unbounded. Defaults to "softplus".
+            bin_embedding_dim (int): bin embedding dimension. Defaults to 128.
+            min_depth (float): Lower bound for normed bin centers. Defaults to 1e-3.
+            max_depth (float): Upper bound for normed bin centers. Defaults to 10.
+            n_attractors (List[int]): Number of bin attractors at decoder layers. Defaults to [16, 8, 4, 1].
+            attractor_alpha (int): Proportional attractor strength. Refer to models.layers.attractor for more details. Defaults to 1000.
+            attractor_gamma (int): Exponential attractor strength. Refer to models.layers.attractor for more details. Defaults to 2.
+            attractor_kind (str): Attraction aggregation "sum" or "mean". Defaults to 'mean'.
+            attractor_type (str): Type of attractor to use; "inv" (Inverse attractor) or "exp" (Exponential attractor). Defaults to 'inv'.
+            min_temp (int): Lower bound for temperature of output probability distribution. Defaults to 0.0212.
+            max_temp (int): Upper bound for temperature of output probability distribution. Defaults to 50.
+            force_keep_ar (bool): If True, the model will keep the aspect ratio of the input image. Defaults to True.
+
+    """
+    if pretrained and midas_model_type != "DPT_BEiT_L_384":
+        raise ValueError(f"Only DPT_BEiT_L_384 MiDaS model is supported for pretrained Zoe_K model, got: {midas_model_type}")
+    
+    if not pretrained:
+        pretrained_resource = None
+    else:
+        pretrained_resource = "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_K.pt"
+
+    config = get_config("zoedepth", config_mode, pretrained_resource=pretrained_resource, config_version="kitti", **kwargs)
+    model = build_model(config)
+    return model
+
+# Zoe_NK
+def ZoeD_NK(pretrained=False, midas_model_type="DPT_BEiT_L_384", config_mode="infer", **kwargs):
+    """ZoeDepthNK model. This is the version of ZoeDepth that has two metric heads and uses a learned router to route to experts.
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on NYU-Depth-V2
+        midas_model_type (str): Midas model type. Should be one of the models as listed in torch.hub.list("intel-isl/MiDaS"). Default: DPT_BEiT_L_384
+    
+    Keyword Args:
+        **kwargs: Additional arguments to pass to the model
+        The following arguments are supported:
+            train_midas (bool): If True, returns a model that with trainable midas base. Defaults to True
+            use_pretrained_midas (bool): If True, returns a model that uses pretrained midas base. Defaults to True
+            bin_conf (List[dict]): A list of dictionaries that contain the bin configuration for each metric head. Each dictionary should contain the following keys: 
+                                    "name" (str, typically same as the dataset name), "n_bins" (int), "min_depth" (float), "max_depth" (float)
+                                   The length of this list determines the number of metric heads.
+            bin_centers_type (str): "normed" or "softplus". Activation type used for bin centers. For "normed" bin centers, linear normalization trick is applied. This results in bounded bin centers.
+                                               For "softplus", softplus activation is used and thus are unbounded. Defaults to "softplus".
+            bin_embedding_dim (int): bin embedding dimension. Defaults to 128.
+
+            n_attractors (List[int]): Number of bin attractors at decoder layers. Defaults to [16, 8, 4, 1].
+            attractor_alpha (int): Proportional attractor strength. Refer to models.layers.attractor for more details. Defaults to 1000.
+            attractor_gamma (int): Exponential attractor strength. Refer to models.layers.attractor for more details. Defaults to 2.
+            attractor_kind (str): Attraction aggregation "sum" or "mean". Defaults to 'mean'.
+            attractor_type (str): Type of attractor to use; "inv" (Inverse attractor) or "exp" (Exponential attractor). Defaults to 'inv'.
+
+            min_temp (int): Lower bound for temperature of output probability distribution. Defaults to 0.0212.
+            max_temp (int): Upper bound for temperature of output probability distribution. Defaults to 50.
+            
+            memory_efficient (bool): Whether to use memory efficient version of attractor layers. Memory efficient version is slower but is recommended incase of multiple metric heads in order save GPU memory. Defaults to True.
+
+    """
+    if pretrained and midas_model_type != "DPT_BEiT_L_384":
+        raise ValueError(f"Only DPT_BEiT_L_384 MiDaS model is supported for pretrained Zoe_NK model, got: {midas_model_type}")
+    
+    if not pretrained:
+        pretrained_resource = None
+    else:
+        pretrained_resource = "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_NK.pt"
+
+    config = get_config("zoedepth_nk", config_mode, pretrained_resource=pretrained_resource, **kwargs)
+    model = build_model(config)
+    return model

ZoeDepth/sanity.py

@@ -0,0 +1,98 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import numpy as np
+from torchvision.transforms import ToTensor
+from PIL import Image
+from zoedepth.utils.misc import get_image_from_url, colorize
+import torch
+
+from zoedepth.models.builder import build_model
+from zoedepth.utils.config import get_config
+from pprint import pprint
+
+
+torch.hub.help("intel-isl/MiDaS", "DPT_BEiT_L_384", force_reload=True) 
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+if DEVICE == "cpu":
+    print("WARNING: Running on CPU. This will be slow. Check your CUDA installation.")
+
+print("*" * 20 + " Testing zoedepth " + "*" * 20)
+conf = get_config("zoedepth", "infer")
+
+
+print("Config:")
+pprint(conf)
+
+model = build_model(conf).to(DEVICE)
+model.eval()
+x = torch.rand(1, 3, 384, 512).to(DEVICE)
+
+print("-"*20 + "Testing on a random input" + "-"*20)
+
+with torch.no_grad():
+    out = model(x)
+
+if isinstance(out, dict):
+    # print shapes of all outputs
+    for k, v in out.items():
+        if v is not None:
+            print(k, v.shape)
+else:
+    print([o.shape for o in out if o is not None])
+
+print("\n\n")
+print("-"*20 + " Testing on an indoor scene from url " + "-"*20)
+
+# Test img
+url = "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS4W8H_Nxk_rs3Vje_zj6mglPOH7bnPhQitBH8WkqjlqQVotdtDEG37BsnGofME3_u6lDk&usqp=CAU"
+img = get_image_from_url(url)
+orig_size = img.size
+X = ToTensor()(img)
+X = X.unsqueeze(0).to(DEVICE)
+
+print("X.shape", X.shape)
+print("predicting")
+
+with torch.no_grad():
+    out = model.infer(X).cpu()
+
+# or just, 
+# out = model.infer_pil(img)
+
+
+print("output.shape", out.shape)
+pred = Image.fromarray(colorize(out))
+# Stack img and pred side by side for comparison and save
+pred = pred.resize(orig_size, Image.ANTIALIAS)
+stacked = Image.new("RGB", (orig_size[0]*2, orig_size[1]))
+stacked.paste(img, (0, 0))
+stacked.paste(pred, (orig_size[0], 0))
+
+stacked.save("pred.png")
+print("saved pred.png")
+
+
+model.infer_pil(img, output_type="pil").save("pred_raw.png")

ZoeDepth/sanity_hub.py

@@ -0,0 +1,43 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import torch
+import numpy as np
+from torchvision.transforms import ToTensor
+from PIL import Image
+from zoedepth.utils.misc import get_image_from_url, colorize
+
+from zoedepth.models.builder import build_model
+from zoedepth.utils.config import get_config
+from pprint import pprint
+
+
+
+# Trigger reload of MiDaS
+torch.hub.help("intel-isl/MiDaS", "DPT_BEiT_L_384", force_reload=True) 
+
+
+model = torch.hub.load(".", "ZoeD_K", source="local", pretrained=True)
+model = torch.hub.load(".", "ZoeD_NK", source="local", pretrained=True)
+model = torch.hub.load(".", "ZoeD_N", source="local", pretrained=True)

ZoeDepth/train_mix.py

@@ -0,0 +1,179 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+from zoedepth.utils.misc import count_parameters, parallelize
+from zoedepth.utils.config import get_config
+from zoedepth.utils.arg_utils import parse_unknown
+from zoedepth.trainers.builder import get_trainer
+from zoedepth.models.builder import build_model
+from zoedepth.data.data_mono import MixedNYUKITTI
+import torch.utils.data.distributed
+import torch.multiprocessing as mp
+import torch
+import numpy as np
+from pprint import pprint
+import argparse
+import os
+
+os.environ["PYOPENGL_PLATFORM"] = "egl"
+os.environ["WANDB_START_METHOD"] = "thread"
+
+
+def fix_random_seed(seed: int):
+    """
+    Fix random seed for reproducibility
+
+    Args:
+        seed (int): random seed
+    """
+    import random
+
+    import numpy
+    import torch
+
+    random.seed(seed)
+    numpy.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+
+def load_ckpt(config, model, checkpoint_dir="./checkpoints", ckpt_type="best"):
+    import glob
+    import os
+
+    from zoedepth.models.model_io import load_wts
+
+    if hasattr(config, "checkpoint"):
+        checkpoint = config.checkpoint
+    elif hasattr(config, "ckpt_pattern"):
+        pattern = config.ckpt_pattern
+        matches = glob.glob(os.path.join(
+            checkpoint_dir, f"*{pattern}*{ckpt_type}*"))
+        if not (len(matches) > 0):
+            raise ValueError(f"No matches found for the pattern {pattern}")
+
+        checkpoint = matches[0]
+
+    else:
+        return model
+    model = load_wts(model, checkpoint)
+    print("Loaded weights from {0}".format(checkpoint))
+    return model
+
+
+def main_worker(gpu, ngpus_per_node, config):
+    try:
+        fix_random_seed(43)
+
+        config.gpu = gpu
+
+        model = build_model(config)
+        model = load_ckpt(config, model)
+        model = parallelize(config, model)
+
+        total_params = f"{round(count_parameters(model)/1e6,2)}M"
+        config.total_params = total_params
+        print(f"Total parameters : {total_params}")
+
+        train_loader = MixedNYUKITTI(config, "train").data
+        test_loader = MixedNYUKITTI(config, "online_eval").data
+
+        trainer = get_trainer(config)(
+            config, model, train_loader, test_loader, device=config.gpu)
+
+        trainer.train()
+    finally:
+        import wandb
+        wandb.finish()
+
+
+if __name__ == '__main__':
+    mp.set_start_method('forkserver')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-m", "--model", type=str, default="synunet")
+    parser.add_argument("-d", "--dataset", type=str, default='mix')
+    parser.add_argument("--trainer", type=str, default=None)
+
+    args, unknown_args = parser.parse_known_args()
+    overwrite_kwargs = parse_unknown(unknown_args)
+
+    overwrite_kwargs["model"] = args.model
+    if args.trainer is not None:
+        overwrite_kwargs["trainer"] = args.trainer
+
+    config = get_config(args.model, "train", args.dataset, **overwrite_kwargs)
+    # git_commit()
+    if config.use_shared_dict:
+        shared_dict = mp.Manager().dict()
+    else:
+        shared_dict = None
+    config.shared_dict = shared_dict
+
+    config.batch_size = config.bs
+    config.mode = 'train'
+    if config.root != "." and not os.path.isdir(config.root):
+        os.makedirs(config.root)
+
+    try:
+        node_str = os.environ['SLURM_JOB_NODELIST'].replace(
+            '[', '').replace(']', '')
+        nodes = node_str.split(',')
+
+        config.world_size = len(nodes)
+        config.rank = int(os.environ['SLURM_PROCID'])
+        # config.save_dir = "/ibex/scratch/bhatsf/videodepth/checkpoints"
+
+    except KeyError as e:
+        # We are NOT using SLURM
+        config.world_size = 1
+        config.rank = 0
+        nodes = ["127.0.0.1"]
+
+    if config.distributed:
+
+        print(config.rank)
+        port = np.random.randint(15000, 15025)
+        config.dist_url = 'tcp://{}:{}'.format(nodes[0], port)
+        print(config.dist_url)
+        config.dist_backend = 'nccl'
+        config.gpu = None
+
+    ngpus_per_node = torch.cuda.device_count()
+    config.num_workers = config.workers
+    config.ngpus_per_node = ngpus_per_node
+    print("Config:")
+    pprint(config)
+    if config.distributed:
+        config.world_size = ngpus_per_node * config.world_size
+        mp.spawn(main_worker, nprocs=ngpus_per_node,
+                 args=(ngpus_per_node, config))
+    else:
+        if ngpus_per_node == 1:
+            config.gpu = 0
+        main_worker(config.gpu, ngpus_per_node, config)

ZoeDepth/train_mono.py

@@ -0,0 +1,174 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+from zoedepth.utils.misc import count_parameters, parallelize
+from zoedepth.utils.config import get_config
+from zoedepth.utils.arg_utils import parse_unknown
+from zoedepth.trainers.builder import get_trainer
+from zoedepth.models.builder import build_model
+from zoedepth.data.data_mono import DepthDataLoader
+import torch.utils.data.distributed
+import torch.multiprocessing as mp
+import torch
+import numpy as np
+from pprint import pprint
+import argparse
+import os
+
+os.environ["PYOPENGL_PLATFORM"] = "egl"
+os.environ["WANDB_START_METHOD"] = "thread"
+
+
+def fix_random_seed(seed: int):
+    import random
+
+    import numpy
+    import torch
+
+    random.seed(seed)
+    numpy.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = True
+
+
+def load_ckpt(config, model, checkpoint_dir="./checkpoints", ckpt_type="best"):
+    import glob
+    import os
+
+    from zoedepth.models.model_io import load_wts
+
+    if hasattr(config, "checkpoint"):
+        checkpoint = config.checkpoint
+    elif hasattr(config, "ckpt_pattern"):
+        pattern = config.ckpt_pattern
+        matches = glob.glob(os.path.join(
+            checkpoint_dir, f"*{pattern}*{ckpt_type}*"))
+        if not (len(matches) > 0):
+            raise ValueError(f"No matches found for the pattern {pattern}")
+
+        checkpoint = matches[0]
+
+    else:
+        return model
+    model = load_wts(model, checkpoint)
+    print("Loaded weights from {0}".format(checkpoint))
+    return model
+
+
+def main_worker(gpu, ngpus_per_node, config):
+    try:
+        seed = config.seed if 'seed' in config and config.seed else 43
+        fix_random_seed(seed)
+
+        config.gpu = gpu
+
+        model = build_model(config)
+        model = load_ckpt(config, model)
+        model = parallelize(config, model)
+
+        total_params = f"{round(count_parameters(model)/1e6,2)}M"
+        config.total_params = total_params
+        print(f"Total parameters : {total_params}")
+
+        train_loader = DepthDataLoader(config, "train").data
+        test_loader = DepthDataLoader(config, "online_eval").data
+
+        trainer = get_trainer(config)(
+            config, model, train_loader, test_loader, device=config.gpu)
+
+        trainer.train()
+    finally:
+        import wandb
+        wandb.finish()
+
+
+if __name__ == '__main__':
+    mp.set_start_method('forkserver')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-m", "--model", type=str, default="synunet")
+    parser.add_argument("-d", "--dataset", type=str, default='nyu')
+    parser.add_argument("--trainer", type=str, default=None)
+
+    args, unknown_args = parser.parse_known_args()
+    overwrite_kwargs = parse_unknown(unknown_args)
+
+    overwrite_kwargs["model"] = args.model
+    if args.trainer is not None:
+        overwrite_kwargs["trainer"] = args.trainer
+
+    config = get_config(args.model, "train", args.dataset, **overwrite_kwargs)
+    # git_commit()
+    if config.use_shared_dict:
+        shared_dict = mp.Manager().dict()
+    else:
+        shared_dict = None
+    config.shared_dict = shared_dict
+
+    config.batch_size = config.bs
+    config.mode = 'train'
+    if config.root != "." and not os.path.isdir(config.root):
+        os.makedirs(config.root)
+
+    try:
+        node_str = os.environ['SLURM_JOB_NODELIST'].replace(
+            '[', '').replace(']', '')
+        nodes = node_str.split(',')
+
+        config.world_size = len(nodes)
+        config.rank = int(os.environ['SLURM_PROCID'])
+        # config.save_dir = "/ibex/scratch/bhatsf/videodepth/checkpoints"
+
+    except KeyError as e:
+        # We are NOT using SLURM
+        config.world_size = 1
+        config.rank = 0
+        nodes = ["127.0.0.1"]
+
+    if config.distributed:
+
+        print(config.rank)
+        port = np.random.randint(15000, 15025)
+        config.dist_url = 'tcp://{}:{}'.format(nodes[0], port)
+        print(config.dist_url)
+        config.dist_backend = 'nccl'
+        config.gpu = None
+
+    ngpus_per_node = torch.cuda.device_count()
+    config.num_workers = config.workers
+    config.ngpus_per_node = ngpus_per_node
+    print("Config:")
+    pprint(config)
+    if config.distributed:
+        config.world_size = ngpus_per_node * config.world_size
+        mp.spawn(main_worker, nprocs=ngpus_per_node,
+                 args=(ngpus_per_node, config))
+    else:
+        if ngpus_per_node == 1:
+            config.gpu = 0
+        main_worker(config.gpu, ngpus_per_node, config)

ZoeDepth/train_test_inputs/nyudepthv2_test_files_with_gt.txt

@@ -0,0 +1,654 @@
+bathroom/rgb_00045.jpg bathroom/sync_depth_00045.png 518.8579
+bathroom/rgb_00046.jpg bathroom/sync_depth_00046.png 518.8579
+bathroom/rgb_00507.jpg bathroom/sync_depth_00507.png 518.8579
+bathroom/rgb_00508.jpg bathroom/sync_depth_00508.png 518.8579
+bathroom/rgb_00509.jpg bathroom/sync_depth_00509.png 518.8579
+bathroom/rgb_00510.jpg bathroom/sync_depth_00510.png 518.8579
+bathroom/rgb_00511.jpg bathroom/sync_depth_00511.png 518.8579
+bathroom/rgb_00512.jpg bathroom/sync_depth_00512.png 518.8579
+bathroom/rgb_00649.jpg bathroom/sync_depth_00649.png 518.8579
+bathroom/rgb_00650.jpg bathroom/sync_depth_00650.png 518.8579
+bathroom/rgb_00655.jpg bathroom/sync_depth_00655.png 518.8579
+bathroom/rgb_00656.jpg bathroom/sync_depth_00656.png 518.8579
+bathroom/rgb_00657.jpg bathroom/sync_depth_00657.png 518.8579
+bathroom/rgb_00662.jpg bathroom/sync_depth_00662.png 518.8579
+bathroom/rgb_00663.jpg bathroom/sync_depth_00663.png 518.8579
+bathroom/rgb_00667.jpg bathroom/sync_depth_00667.png 518.8579
+bathroom/rgb_00668.jpg bathroom/sync_depth_00668.png 518.8579
+bathroom/rgb_00670.jpg bathroom/sync_depth_00670.png 518.8579
+bathroom/rgb_00671.jpg bathroom/sync_depth_00671.png 518.8579
+bathroom/rgb_00672.jpg bathroom/sync_depth_00672.png 518.8579
+bathroom/rgb_00675.jpg bathroom/sync_depth_00675.png 518.8579
+bathroom/rgb_00676.jpg bathroom/sync_depth_00676.png 518.8579
+bathroom/rgb_00677.jpg bathroom/sync_depth_00677.png 518.8579
+bathroom/rgb_00678.jpg bathroom/sync_depth_00678.png 518.8579
+bathroom/rgb_00679.jpg bathroom/sync_depth_00679.png 518.8579
+bathroom/rgb_00680.jpg bathroom/sync_depth_00680.png 518.8579
+bathroom/rgb_00685.jpg bathroom/sync_depth_00685.png 518.8579
+bathroom/rgb_00686.jpg bathroom/sync_depth_00686.png 518.8579
+bathroom/rgb_00687.jpg bathroom/sync_depth_00687.png 518.8579
+bathroom/rgb_00688.jpg bathroom/sync_depth_00688.png 518.8579
+bathroom/rgb_00689.jpg bathroom/sync_depth_00689.png 518.8579
+bathroom/rgb_00692.jpg bathroom/sync_depth_00692.png 518.8579
+bathroom/rgb_00693.jpg bathroom/sync_depth_00693.png 518.8579
+bathroom/rgb_00696.jpg bathroom/sync_depth_00696.png 518.8579
+bathroom/rgb_00669.jpg bathroom/sync_depth_00669.png 518.8579
+bathroom/rgb_00697.jpg bathroom/sync_depth_00697.png 518.8579
+bathroom/rgb_00698.jpg bathroom/sync_depth_00698.png 518.8579
+bathroom/rgb_00705.jpg bathroom/sync_depth_00705.png 518.8579
+bathroom/rgb_00706.jpg bathroom/sync_depth_00706.png 518.8579
+bathroom/rgb_00707.jpg bathroom/sync_depth_00707.png 518.8579
+bathroom/rgb_00708.jpg bathroom/sync_depth_00708.png 518.8579
+bathroom/rgb_00709.jpg bathroom/sync_depth_00709.png 518.8579
+bathroom/rgb_00710.jpg bathroom/sync_depth_00710.png 518.8579
+bathroom/rgb_00711.jpg bathroom/sync_depth_00711.png 518.8579
+bathroom/rgb_00712.jpg bathroom/sync_depth_00712.png 518.8579
+bathroom/rgb_00716.jpg bathroom/sync_depth_00716.png 518.8579
+bathroom/rgb_00717.jpg bathroom/sync_depth_00717.png 518.8579
+bathroom/rgb_00723.jpg bathroom/sync_depth_00723.png 518.8579
+bathroom/rgb_00724.jpg bathroom/sync_depth_00724.png 518.8579
+bathroom/rgb_00725.jpg bathroom/sync_depth_00725.png 518.8579
+bathroom/rgb_00726.jpg bathroom/sync_depth_00726.png 518.8579
+bathroom/rgb_00727.jpg bathroom/sync_depth_00727.png 518.8579
+bathroom/rgb_00730.jpg bathroom/sync_depth_00730.png 518.8579
+bathroom/rgb_00731.jpg bathroom/sync_depth_00731.png 518.8579
+bathroom/rgb_00732.jpg bathroom/sync_depth_00732.png 518.8579
+bathroom/rgb_00733.jpg bathroom/sync_depth_00733.png 518.8579
+bathroom/rgb_00742.jpg bathroom/sync_depth_00742.png 518.8579
+bathroom/rgb_00743.jpg bathroom/sync_depth_00743.png 518.8579
+bedroom/rgb_00055.jpg bedroom/sync_depth_00055.png 518.8579
+bedroom/rgb_00056.jpg bedroom/sync_depth_00056.png 518.8579
+bedroom/rgb_00058.jpg bedroom/sync_depth_00058.png 518.8579
+bedroom/rgb_00059.jpg bedroom/sync_depth_00059.png 518.8579
+bedroom/rgb_00060.jpg bedroom/sync_depth_00060.png 518.8579
+bedroom/rgb_00061.jpg bedroom/sync_depth_00061.png 518.8579
+bedroom/rgb_00062.jpg bedroom/sync_depth_00062.png 518.8579
+bedroom/rgb_00075.jpg bedroom/sync_depth_00075.png 518.8579
+bedroom/rgb_00076.jpg bedroom/sync_depth_00076.png 518.8579
+bedroom/rgb_00077.jpg bedroom/sync_depth_00077.png 518.8579
+bedroom/rgb_00078.jpg bedroom/sync_depth_00078.png 518.8579
+bedroom/rgb_00170.jpg bedroom/sync_depth_00170.png 518.8579
+bedroom/rgb_00171.jpg bedroom/sync_depth_00171.png 518.8579
+bedroom/rgb_00172.jpg bedroom/sync_depth_00172.png 518.8579
+bedroom/rgb_00173.jpg bedroom/sync_depth_00173.png 518.8579
+bedroom/rgb_00174.jpg bedroom/sync_depth_00174.png 518.8579
+bedroom/rgb_00175.jpg bedroom/sync_depth_00175.png 518.8579
+bedroom/rgb_00180.jpg bedroom/sync_depth_00180.png 518.8579
+bedroom/rgb_00181.jpg bedroom/sync_depth_00181.png 518.8579
+bedroom/rgb_00182.jpg bedroom/sync_depth_00182.png 518.8579
+bedroom/rgb_00183.jpg bedroom/sync_depth_00183.png 518.8579
+bedroom/rgb_00184.jpg bedroom/sync_depth_00184.png 518.8579
+bedroom/rgb_00185.jpg bedroom/sync_depth_00185.png 518.8579
+bedroom/rgb_00186.jpg bedroom/sync_depth_00186.png 518.8579
+bedroom/rgb_00187.jpg bedroom/sync_depth_00187.png 518.8579
+bedroom/rgb_00188.jpg bedroom/sync_depth_00188.png 518.8579
+bedroom/rgb_00189.jpg bedroom/sync_depth_00189.png 518.8579
+bedroom/rgb_00190.jpg bedroom/sync_depth_00190.png 518.8579
+bedroom/rgb_00191.jpg bedroom/sync_depth_00191.png 518.8579
+bedroom/rgb_00192.jpg bedroom/sync_depth_00192.png 518.8579
+bedroom/rgb_00219.jpg bedroom/sync_depth_00219.png 518.8579
+bedroom/rgb_00220.jpg bedroom/sync_depth_00220.png 518.8579
+bedroom/rgb_00221.jpg bedroom/sync_depth_00221.png 518.8579
+bedroom/rgb_00279.jpg bedroom/sync_depth_00279.png 518.8579
+bedroom/rgb_00179.jpg bedroom/sync_depth_00179.png 518.8579
+bedroom/rgb_00280.jpg bedroom/sync_depth_00280.png 518.8579
+bedroom/rgb_00536.jpg bedroom/sync_depth_00536.png 518.8579
+bedroom/rgb_00960.jpg bedroom/sync_depth_00960.png 518.8579
+bedroom/rgb_01000.jpg bedroom/sync_depth_01000.png 518.8579
+bedroom/rgb_01052.jpg bedroom/sync_depth_01052.png 518.8579
+bedroom/rgb_01092.jpg bedroom/sync_depth_01092.png 518.8579
+bedroom/rgb_01122.jpg bedroom/sync_depth_01122.png 518.8579
+bedroom/rgb_01150.jpg bedroom/sync_depth_01150.png 518.8579
+bedroom/rgb_00281.jpg bedroom/sync_depth_00281.png 518.8579
+bedroom/rgb_00282.jpg bedroom/sync_depth_00282.png 518.8579
+bedroom/rgb_00514.jpg bedroom/sync_depth_00514.png 518.8579
+bedroom/rgb_00515.jpg bedroom/sync_depth_00515.png 518.8579
+bedroom/rgb_00516.jpg bedroom/sync_depth_00516.png 518.8579
+bedroom/rgb_00517.jpg bedroom/sync_depth_00517.png 518.8579
+bedroom/rgb_00518.jpg bedroom/sync_depth_00518.png 518.8579
+bedroom/rgb_00519.jpg bedroom/sync_depth_00519.png 518.8579
+bedroom/rgb_00520.jpg bedroom/sync_depth_00520.png 518.8579
+bedroom/rgb_00521.jpg bedroom/sync_depth_00521.png 518.8579
+bedroom/rgb_00522.jpg bedroom/sync_depth_00522.png 518.8579
+bedroom/rgb_00523.jpg bedroom/sync_depth_00523.png 518.8579
+bedroom/rgb_00524.jpg bedroom/sync_depth_00524.png 518.8579
+bedroom/rgb_00525.jpg bedroom/sync_depth_00525.png 518.8579
+bedroom/rgb_00530.jpg bedroom/sync_depth_00530.png 518.8579
+bedroom/rgb_00531.jpg bedroom/sync_depth_00531.png 518.8579
+bedroom/rgb_00532.jpg bedroom/sync_depth_00532.png 518.8579
+bedroom/rgb_00537.jpg bedroom/sync_depth_00537.png 518.8579
+bedroom/rgb_00538.jpg bedroom/sync_depth_00538.png 518.8579
+bedroom/rgb_00916.jpg bedroom/sync_depth_00916.png 518.8579
+bedroom/rgb_00917.jpg bedroom/sync_depth_00917.png 518.8579
+bedroom/rgb_00918.jpg bedroom/sync_depth_00918.png 518.8579
+bedroom/rgb_00925.jpg bedroom/sync_depth_00925.png 518.8579
+bedroom/rgb_00926.jpg bedroom/sync_depth_00926.png 518.8579
+bedroom/rgb_00927.jpg bedroom/sync_depth_00927.png 518.8579
+bedroom/rgb_00931.jpg bedroom/sync_depth_00931.png 518.8579
+bedroom/rgb_00932.jpg bedroom/sync_depth_00932.png 518.8579
+bedroom/rgb_00933.jpg bedroom/sync_depth_00933.png 518.8579
+bedroom/rgb_00934.jpg bedroom/sync_depth_00934.png 518.8579
+bedroom/rgb_00944.jpg bedroom/sync_depth_00944.png 518.8579
+bedroom/rgb_00945.jpg bedroom/sync_depth_00945.png 518.8579
+bedroom/rgb_00946.jpg bedroom/sync_depth_00946.png 518.8579
+bedroom/rgb_00958.jpg bedroom/sync_depth_00958.png 518.8579
+bedroom/rgb_00959.jpg bedroom/sync_depth_00959.png 518.8579
+bedroom/rgb_00961.jpg bedroom/sync_depth_00961.png 518.8579
+bedroom/rgb_00964.jpg bedroom/sync_depth_00964.png 518.8579
+bedroom/rgb_00965.jpg bedroom/sync_depth_00965.png 518.8579
+bedroom/rgb_00966.jpg bedroom/sync_depth_00966.png 518.8579
+bedroom/rgb_00969.jpg bedroom/sync_depth_00969.png 518.8579
+bedroom/rgb_00970.jpg bedroom/sync_depth_00970.png 518.8579
+bedroom/rgb_00971.jpg bedroom/sync_depth_00971.png 518.8579
+bedroom/rgb_00972.jpg bedroom/sync_depth_00972.png 518.8579
+bedroom/rgb_00973.jpg bedroom/sync_depth_00973.png 518.8579
+bedroom/rgb_00974.jpg bedroom/sync_depth_00974.png 518.8579
+bedroom/rgb_00975.jpg bedroom/sync_depth_00975.png 518.8579
+bedroom/rgb_00976.jpg bedroom/sync_depth_00976.png 518.8579
+bedroom/rgb_00990.jpg bedroom/sync_depth_00990.png 518.8579
+bedroom/rgb_00991.jpg bedroom/sync_depth_00991.png 518.8579
+bedroom/rgb_00992.jpg bedroom/sync_depth_00992.png 518.8579
+bedroom/rgb_00993.jpg bedroom/sync_depth_00993.png 518.8579
+bedroom/rgb_00994.jpg bedroom/sync_depth_00994.png 518.8579
+bedroom/rgb_01001.jpg bedroom/sync_depth_01001.png 518.8579
+bedroom/rgb_01002.jpg bedroom/sync_depth_01002.png 518.8579
+bedroom/rgb_01003.jpg bedroom/sync_depth_01003.png 518.8579
+bedroom/rgb_01009.jpg bedroom/sync_depth_01009.png 518.8579
+bedroom/rgb_01010.jpg bedroom/sync_depth_01010.png 518.8579
+bedroom/rgb_01011.jpg bedroom/sync_depth_01011.png 518.8579
+bedroom/rgb_01020.jpg bedroom/sync_depth_01020.png 518.8579
+bedroom/rgb_01021.jpg bedroom/sync_depth_01021.png 518.8579
+bedroom/rgb_01022.jpg bedroom/sync_depth_01022.png 518.8579
+bedroom/rgb_01031.jpg bedroom/sync_depth_01031.png 518.8579
+bedroom/rgb_01032.jpg bedroom/sync_depth_01032.png 518.8579
+bedroom/rgb_01033.jpg bedroom/sync_depth_01033.png 518.8579
+bedroom/rgb_01037.jpg bedroom/sync_depth_01037.png 518.8579
+bedroom/rgb_01038.jpg bedroom/sync_depth_01038.png 518.8579
+bedroom/rgb_01047.jpg bedroom/sync_depth_01047.png 518.8579
+bedroom/rgb_01048.jpg bedroom/sync_depth_01048.png 518.8579
+bedroom/rgb_01051.jpg bedroom/sync_depth_01051.png 518.8579
+bedroom/rgb_01056.jpg bedroom/sync_depth_01056.png 518.8579
+bedroom/rgb_01057.jpg bedroom/sync_depth_01057.png 518.8579
+bedroom/rgb_01074.jpg bedroom/sync_depth_01074.png 518.8579
+bedroom/rgb_01075.jpg bedroom/sync_depth_01075.png 518.8579
+bedroom/rgb_01076.jpg bedroom/sync_depth_01076.png 518.8579
+bedroom/rgb_01077.jpg bedroom/sync_depth_01077.png 518.8579
+bedroom/rgb_01078.jpg bedroom/sync_depth_01078.png 518.8579
+bedroom/rgb_01079.jpg bedroom/sync_depth_01079.png 518.8579
+bedroom/rgb_01080.jpg bedroom/sync_depth_01080.png 518.8579
+bedroom/rgb_01081.jpg bedroom/sync_depth_01081.png 518.8579
+bedroom/rgb_01082.jpg bedroom/sync_depth_01082.png 518.8579
+bedroom/rgb_01083.jpg bedroom/sync_depth_01083.png 518.8579
+bedroom/rgb_01087.jpg bedroom/sync_depth_01087.png 518.8579
+bedroom/rgb_01088.jpg bedroom/sync_depth_01088.png 518.8579
+bedroom/rgb_01089.jpg bedroom/sync_depth_01089.png 518.8579
+bedroom/rgb_01090.jpg bedroom/sync_depth_01090.png 518.8579
+bedroom/rgb_01091.jpg bedroom/sync_depth_01091.png 518.8579
+bedroom/rgb_01093.jpg bedroom/sync_depth_01093.png 518.8579
+bedroom/rgb_01094.jpg bedroom/sync_depth_01094.png 518.8579
+bedroom/rgb_01095.jpg bedroom/sync_depth_01095.png 518.8579
+bedroom/rgb_01097.jpg bedroom/sync_depth_01097.png 518.8579
+bedroom/rgb_01098.jpg bedroom/sync_depth_01098.png 518.8579
+bedroom/rgb_01099.jpg bedroom/sync_depth_01099.png 518.8579
+bedroom/rgb_01100.jpg bedroom/sync_depth_01100.png 518.8579
+bedroom/rgb_01101.jpg bedroom/sync_depth_01101.png 518.8579
+bedroom/rgb_01102.jpg bedroom/sync_depth_01102.png 518.8579
+bedroom/rgb_01103.jpg bedroom/sync_depth_01103.png 518.8579
+bedroom/rgb_01105.jpg bedroom/sync_depth_01105.png 518.8579
+bedroom/rgb_01106.jpg bedroom/sync_depth_01106.png 518.8579
+bedroom/rgb_01107.jpg bedroom/sync_depth_01107.png 518.8579
+bedroom/rgb_01108.jpg bedroom/sync_depth_01108.png 518.8579
+bedroom/rgb_01116.jpg bedroom/sync_depth_01116.png 518.8579
+bedroom/rgb_01117.jpg bedroom/sync_depth_01117.png 518.8579
+bedroom/rgb_01118.jpg bedroom/sync_depth_01118.png 518.8579
+bedroom/rgb_01123.jpg bedroom/sync_depth_01123.png 518.8579
+bedroom/rgb_01124.jpg bedroom/sync_depth_01124.png 518.8579
+bedroom/rgb_01125.jpg bedroom/sync_depth_01125.png 518.8579
+bedroom/rgb_01126.jpg bedroom/sync_depth_01126.png 518.8579
+bedroom/rgb_01127.jpg bedroom/sync_depth_01127.png 518.8579
+bedroom/rgb_01128.jpg bedroom/sync_depth_01128.png 518.8579
+bedroom/rgb_01129.jpg bedroom/sync_depth_01129.png 518.8579
+bedroom/rgb_01130.jpg bedroom/sync_depth_01130.png 518.8579
+bedroom/rgb_01134.jpg bedroom/sync_depth_01134.png 518.8579
+bedroom/rgb_01135.jpg bedroom/sync_depth_01135.png 518.8579
+bedroom/rgb_01143.jpg bedroom/sync_depth_01143.png 518.8579
+bedroom/rgb_01144.jpg bedroom/sync_depth_01144.png 518.8579
+bedroom/rgb_01145.jpg bedroom/sync_depth_01145.png 518.8579
+bedroom/rgb_01146.jpg bedroom/sync_depth_01146.png 518.8579
+bedroom/rgb_01147.jpg bedroom/sync_depth_01147.png 518.8579
+bedroom/rgb_01148.jpg bedroom/sync_depth_01148.png 518.8579
+bedroom/rgb_01149.jpg bedroom/sync_depth_01149.png 518.8579
+bedroom/rgb_01151.jpg bedroom/sync_depth_01151.png 518.8579
+bedroom/rgb_01152.jpg bedroom/sync_depth_01152.png 518.8579
+bedroom/rgb_01153.jpg bedroom/sync_depth_01153.png 518.8579
+bedroom/rgb_01154.jpg bedroom/sync_depth_01154.png 518.8579
+bedroom/rgb_01155.jpg bedroom/sync_depth_01155.png 518.8579
+bedroom/rgb_01156.jpg bedroom/sync_depth_01156.png 518.8579
+bedroom/rgb_01157.jpg bedroom/sync_depth_01157.png 518.8579
+bedroom/rgb_01161.jpg bedroom/sync_depth_01161.png 518.8579
+bedroom/rgb_01162.jpg bedroom/sync_depth_01162.png 518.8579
+bedroom/rgb_01163.jpg bedroom/sync_depth_01163.png 518.8579
+bedroom/rgb_01164.jpg bedroom/sync_depth_01164.png 518.8579
+bedroom/rgb_01165.jpg bedroom/sync_depth_01165.png 518.8579
+bedroom/rgb_01166.jpg bedroom/sync_depth_01166.png 518.8579
+bedroom/rgb_01169.jpg bedroom/sync_depth_01169.png 518.8579
+bedroom/rgb_01170.jpg bedroom/sync_depth_01170.png 518.8579
+bedroom/rgb_01173.jpg bedroom/sync_depth_01173.png 518.8579
+bedroom/rgb_01174.jpg bedroom/sync_depth_01174.png 518.8579
+bedroom/rgb_01175.jpg bedroom/sync_depth_01175.png 518.8579
+bedroom/rgb_01178.jpg bedroom/sync_depth_01178.png 518.8579
+bedroom/rgb_01179.jpg bedroom/sync_depth_01179.png 518.8579
+bedroom/rgb_01180.jpg bedroom/sync_depth_01180.png 518.8579
+bedroom/rgb_01181.jpg bedroom/sync_depth_01181.png 518.8579
+bedroom/rgb_01182.jpg bedroom/sync_depth_01182.png 518.8579
+bedroom/rgb_01183.jpg bedroom/sync_depth_01183.png 518.8579
+bedroom/rgb_01191.jpg bedroom/sync_depth_01191.png 518.8579
+bedroom/rgb_01192.jpg bedroom/sync_depth_01192.png 518.8579
+bedroom/rgb_01193.jpg bedroom/sync_depth_01193.png 518.8579
+bedroom/rgb_01194.jpg bedroom/sync_depth_01194.png 518.8579
+bedroom/rgb_01195.jpg bedroom/sync_depth_01195.png 518.8579
+bookstore/rgb_00083.jpg bookstore/sync_depth_00083.png 518.8579
+bookstore/rgb_00084.jpg bookstore/sync_depth_00084.png 518.8579
+bookstore/rgb_00085.jpg bookstore/sync_depth_00085.png 518.8579
+bookstore/rgb_00086.jpg bookstore/sync_depth_00086.png 518.8579
+bookstore/rgb_00087.jpg bookstore/sync_depth_00087.png 518.8579
+bookstore/rgb_00088.jpg bookstore/sync_depth_00088.png 518.8579
+bookstore/rgb_00089.jpg bookstore/sync_depth_00089.png 518.8579
+bookstore/rgb_00090.jpg bookstore/sync_depth_00090.png 518.8579
+bookstore/rgb_00116.jpg bookstore/sync_depth_00116.png 518.8579
+bookstore/rgb_00117.jpg bookstore/sync_depth_00117.png 518.8579
+bookstore/rgb_00118.jpg bookstore/sync_depth_00118.png 518.8579
+classroom/rgb_00283.jpg classroom/sync_depth_00283.png 518.8579
+classroom/rgb_00284.jpg classroom/sync_depth_00284.png 518.8579
+classroom/rgb_00295.jpg classroom/sync_depth_00295.png 518.8579
+classroom/rgb_00296.jpg classroom/sync_depth_00296.png 518.8579
+classroom/rgb_00297.jpg classroom/sync_depth_00297.png 518.8579
+classroom/rgb_00298.jpg classroom/sync_depth_00298.png 518.8579
+classroom/rgb_00299.jpg classroom/sync_depth_00299.png 518.8579
+classroom/rgb_00300.jpg classroom/sync_depth_00300.png 518.8579
+classroom/rgb_00301.jpg classroom/sync_depth_00301.png 518.8579
+classroom/rgb_00309.jpg classroom/sync_depth_00309.png 518.8579
+classroom/rgb_00310.jpg classroom/sync_depth_00310.png 518.8579
+classroom/rgb_00311.jpg classroom/sync_depth_00311.png 518.8579
+classroom/rgb_00314.jpg classroom/sync_depth_00314.png 518.8579
+classroom/rgb_00315.jpg classroom/sync_depth_00315.png 518.8579
+classroom/rgb_00316.jpg classroom/sync_depth_00316.png 518.8579
+classroom/rgb_00324.jpg classroom/sync_depth_00324.png 518.8579
+classroom/rgb_00325.jpg classroom/sync_depth_00325.png 518.8579
+classroom/rgb_00326.jpg classroom/sync_depth_00326.png 518.8579
+classroom/rgb_00327.jpg classroom/sync_depth_00327.png 518.8579
+classroom/rgb_00328.jpg classroom/sync_depth_00328.png 518.8579
+classroom/rgb_00329.jpg classroom/sync_depth_00329.png 518.8579
+classroom/rgb_00330.jpg classroom/sync_depth_00330.png 518.8579
+classroom/rgb_00331.jpg classroom/sync_depth_00331.png 518.8579
+computer_lab/rgb_00332.jpg computer_lab/sync_depth_00332.png 518.8579
+computer_lab/rgb_00333.jpg computer_lab/sync_depth_00333.png 518.8579
+computer_lab/rgb_00334.jpg computer_lab/sync_depth_00334.png 518.8579
+dining_room/rgb_00548.jpg dining_room/sync_depth_00548.png 518.8579
+dining_room/rgb_00549.jpg dining_room/sync_depth_00549.png 518.8579
+dining_room/rgb_00550.jpg dining_room/sync_depth_00550.png 518.8579
+dining_room/rgb_01346.jpg dining_room/sync_depth_01346.png 518.8579
+dining_room/rgb_01347.jpg dining_room/sync_depth_01347.png 518.8579
+dining_room/rgb_01348.jpg dining_room/sync_depth_01348.png 518.8579
+dining_room/rgb_01352.jpg dining_room/sync_depth_01352.png 518.8579
+dining_room/rgb_01353.jpg dining_room/sync_depth_01353.png 518.8579
+dining_room/rgb_01354.jpg dining_room/sync_depth_01354.png 518.8579
+dining_room/rgb_01355.jpg dining_room/sync_depth_01355.png 518.8579
+dining_room/rgb_01363.jpg dining_room/sync_depth_01363.png 518.8579
+dining_room/rgb_01364.jpg dining_room/sync_depth_01364.png 518.8579
+dining_room/rgb_01367.jpg dining_room/sync_depth_01367.png 518.8579
+dining_room/rgb_01368.jpg dining_room/sync_depth_01368.png 518.8579
+dining_room/rgb_01383.jpg dining_room/sync_depth_01383.png 518.8579
+dining_room/rgb_01384.jpg dining_room/sync_depth_01384.png 518.8579
+dining_room/rgb_01385.jpg dining_room/sync_depth_01385.png 518.8579
+dining_room/rgb_01387.jpg dining_room/sync_depth_01387.png 518.8579
+dining_room/rgb_01388.jpg dining_room/sync_depth_01388.png 518.8579
+dining_room/rgb_01389.jpg dining_room/sync_depth_01389.png 518.8579
+dining_room/rgb_01390.jpg dining_room/sync_depth_01390.png 518.8579
+dining_room/rgb_01393.jpg dining_room/sync_depth_01393.png 518.8579
+dining_room/rgb_01394.jpg dining_room/sync_depth_01394.png 518.8579
+dining_room/rgb_01395.jpg dining_room/sync_depth_01395.png 518.8579
+dining_room/rgb_01396.jpg dining_room/sync_depth_01396.png 518.8579
+dining_room/rgb_01397.jpg dining_room/sync_depth_01397.png 518.8579
+dining_room/rgb_01398.jpg dining_room/sync_depth_01398.png 518.8579
+dining_room/rgb_01399.jpg dining_room/sync_depth_01399.png 518.8579
+dining_room/rgb_01400.jpg dining_room/sync_depth_01400.png 518.8579
+dining_room/rgb_01406.jpg dining_room/sync_depth_01406.png 518.8579
+dining_room/rgb_01407.jpg dining_room/sync_depth_01407.png 518.8579
+dining_room/rgb_01408.jpg dining_room/sync_depth_01408.png 518.8579
+dining_room/rgb_01409.jpg dining_room/sync_depth_01409.png 518.8579
+dining_room/rgb_01410.jpg dining_room/sync_depth_01410.png 518.8579
+dining_room/rgb_01386.jpg dining_room/sync_depth_01386.png 518.8579
+dining_room/rgb_01411.jpg dining_room/sync_depth_01411.png 518.8579
+dining_room/rgb_01412.jpg dining_room/sync_depth_01412.png 518.8579
+dining_room/rgb_01413.jpg dining_room/sync_depth_01413.png 518.8579
+dining_room/rgb_01420.jpg dining_room/sync_depth_01420.png 518.8579
+dining_room/rgb_01421.jpg dining_room/sync_depth_01421.png 518.8579
+dining_room/rgb_01422.jpg dining_room/sync_depth_01422.png 518.8579
+dining_room/rgb_01423.jpg dining_room/sync_depth_01423.png 518.8579
+dining_room/rgb_01429.jpg dining_room/sync_depth_01429.png 518.8579
+dining_room/rgb_01430.jpg dining_room/sync_depth_01430.png 518.8579
+dining_room/rgb_01431.jpg dining_room/sync_depth_01431.png 518.8579
+dining_room/rgb_01432.jpg dining_room/sync_depth_01432.png 518.8579
+dining_room/rgb_01440.jpg dining_room/sync_depth_01440.png 518.8579
+dining_room/rgb_01441.jpg dining_room/sync_depth_01441.png 518.8579
+dining_room/rgb_01442.jpg dining_room/sync_depth_01442.png 518.8579
+dining_room/rgb_01443.jpg dining_room/sync_depth_01443.png 518.8579
+dining_room/rgb_01444.jpg dining_room/sync_depth_01444.png 518.8579
+dining_room/rgb_01445.jpg dining_room/sync_depth_01445.png 518.8579
+dining_room/rgb_01446.jpg dining_room/sync_depth_01446.png 518.8579
+dining_room/rgb_01447.jpg dining_room/sync_depth_01447.png 518.8579
+dining_room/rgb_01448.jpg dining_room/sync_depth_01448.png 518.8579
+foyer/rgb_00350.jpg foyer/sync_depth_00350.png 518.8579
+foyer/rgb_00351.jpg foyer/sync_depth_00351.png 518.8579
+home_office/rgb_00354.jpg home_office/sync_depth_00354.png 518.8579
+home_office/rgb_00355.jpg home_office/sync_depth_00355.png 518.8579
+home_office/rgb_00356.jpg home_office/sync_depth_00356.png 518.8579
+home_office/rgb_00357.jpg home_office/sync_depth_00357.png 518.8579
+home_office/rgb_00358.jpg home_office/sync_depth_00358.png 518.8579
+home_office/rgb_00359.jpg home_office/sync_depth_00359.png 518.8579
+home_office/rgb_00360.jpg home_office/sync_depth_00360.png 518.8579
+home_office/rgb_00361.jpg home_office/sync_depth_00361.png 518.8579
+home_office/rgb_00362.jpg home_office/sync_depth_00362.png 518.8579
+home_office/rgb_00363.jpg home_office/sync_depth_00363.png 518.8579
+home_office/rgb_00383.jpg home_office/sync_depth_00383.png 518.8579
+home_office/rgb_00384.jpg home_office/sync_depth_00384.png 518.8579
+home_office/rgb_00385.jpg home_office/sync_depth_00385.png 518.8579
+home_office/rgb_00386.jpg home_office/sync_depth_00386.png 518.8579
+home_office/rgb_00387.jpg home_office/sync_depth_00387.png 518.8579
+home_office/rgb_00388.jpg home_office/sync_depth_00388.png 518.8579
+home_office/rgb_00389.jpg home_office/sync_depth_00389.png 518.8579
+home_office/rgb_00394.jpg home_office/sync_depth_00394.png 518.8579
+home_office/rgb_00395.jpg home_office/sync_depth_00395.png 518.8579
+home_office/rgb_00396.jpg home_office/sync_depth_00396.png 518.8579
+home_office/rgb_00554.jpg home_office/sync_depth_00554.png 518.8579
+home_office/rgb_00555.jpg home_office/sync_depth_00555.png 518.8579
+home_office/rgb_00556.jpg home_office/sync_depth_00556.png 518.8579
+home_office/rgb_00557.jpg home_office/sync_depth_00557.png 518.8579
+kitchen/rgb_00000.jpg kitchen/sync_depth_00000.png 518.8579
+kitchen/rgb_00001.jpg kitchen/sync_depth_00001.png 518.8579
+kitchen/rgb_00124.jpg kitchen/sync_depth_00124.png 518.8579
+kitchen/rgb_00125.jpg kitchen/sync_depth_00125.png 518.8579
+kitchen/rgb_00126.jpg kitchen/sync_depth_00126.png 518.8579
+kitchen/rgb_00127.jpg kitchen/sync_depth_00127.png 518.8579
+kitchen/rgb_00128.jpg kitchen/sync_depth_00128.png 518.8579
+kitchen/rgb_00130.jpg kitchen/sync_depth_00130.png 518.8579
+kitchen/rgb_00131.jpg kitchen/sync_depth_00131.png 518.8579
+kitchen/rgb_00132.jpg kitchen/sync_depth_00132.png 518.8579
+kitchen/rgb_00133.jpg kitchen/sync_depth_00133.png 518.8579
+kitchen/rgb_00136.jpg kitchen/sync_depth_00136.png 518.8579
+kitchen/rgb_00193.jpg kitchen/sync_depth_00193.png 518.8579
+kitchen/rgb_00194.jpg kitchen/sync_depth_00194.png 518.8579
+kitchen/rgb_00195.jpg kitchen/sync_depth_00195.png 518.8579
+kitchen/rgb_00196.jpg kitchen/sync_depth_00196.png 518.8579
+kitchen/rgb_00197.jpg kitchen/sync_depth_00197.png 518.8579
+kitchen/rgb_00199.jpg kitchen/sync_depth_00199.png 518.8579
+kitchen/rgb_00200.jpg kitchen/sync_depth_00200.png 518.8579
+kitchen/rgb_00201.jpg kitchen/sync_depth_00201.png 518.8579
+kitchen/rgb_00249.jpg kitchen/sync_depth_00249.png 518.8579
+kitchen/rgb_00558.jpg kitchen/sync_depth_00558.png 518.8579
+kitchen/rgb_00559.jpg kitchen/sync_depth_00559.png 518.8579
+kitchen/rgb_00560.jpg kitchen/sync_depth_00560.png 518.8579
+kitchen/rgb_00561.jpg kitchen/sync_depth_00561.png 518.8579
+kitchen/rgb_00562.jpg kitchen/sync_depth_00562.png 518.8579
+kitchen/rgb_00563.jpg kitchen/sync_depth_00563.png 518.8579
+kitchen/rgb_00564.jpg kitchen/sync_depth_00564.png 518.8579
+kitchen/rgb_00565.jpg kitchen/sync_depth_00565.png 518.8579
+kitchen/rgb_00566.jpg kitchen/sync_depth_00566.png 518.8579
+kitchen/rgb_00567.jpg kitchen/sync_depth_00567.png 518.8579
+kitchen/rgb_00568.jpg kitchen/sync_depth_00568.png 518.8579
+kitchen/rgb_00569.jpg kitchen/sync_depth_00569.png 518.8579
+kitchen/rgb_00570.jpg kitchen/sync_depth_00570.png 518.8579
+kitchen/rgb_00198.jpg kitchen/sync_depth_00198.png 518.8579
+kitchen/rgb_00758.jpg kitchen/sync_depth_00758.png 518.8579
+kitchen/rgb_00776.jpg kitchen/sync_depth_00776.png 518.8579
+kitchen/rgb_00811.jpg kitchen/sync_depth_00811.png 518.8579
+kitchen/rgb_00844.jpg kitchen/sync_depth_00844.png 518.8579
+kitchen/rgb_00759.jpg kitchen/sync_depth_00759.png 518.8579
+kitchen/rgb_00760.jpg kitchen/sync_depth_00760.png 518.8579
+kitchen/rgb_00761.jpg kitchen/sync_depth_00761.png 518.8579
+kitchen/rgb_00762.jpg kitchen/sync_depth_00762.png 518.8579
+kitchen/rgb_00763.jpg kitchen/sync_depth_00763.png 518.8579
+kitchen/rgb_00764.jpg kitchen/sync_depth_00764.png 518.8579
+kitchen/rgb_00765.jpg kitchen/sync_depth_00765.png 518.8579
+kitchen/rgb_00766.jpg kitchen/sync_depth_00766.png 518.8579
+kitchen/rgb_00767.jpg kitchen/sync_depth_00767.png 518.8579
+kitchen/rgb_00768.jpg kitchen/sync_depth_00768.png 518.8579
+kitchen/rgb_00769.jpg kitchen/sync_depth_00769.png 518.8579
+kitchen/rgb_00770.jpg kitchen/sync_depth_00770.png 518.8579
+kitchen/rgb_00771.jpg kitchen/sync_depth_00771.png 518.8579
+kitchen/rgb_00772.jpg kitchen/sync_depth_00772.png 518.8579
+kitchen/rgb_00773.jpg kitchen/sync_depth_00773.png 518.8579
+kitchen/rgb_00774.jpg kitchen/sync_depth_00774.png 518.8579
+kitchen/rgb_00775.jpg kitchen/sync_depth_00775.png 518.8579
+kitchen/rgb_00777.jpg kitchen/sync_depth_00777.png 518.8579
+kitchen/rgb_00778.jpg kitchen/sync_depth_00778.png 518.8579
+kitchen/rgb_00779.jpg kitchen/sync_depth_00779.png 518.8579
+kitchen/rgb_00780.jpg kitchen/sync_depth_00780.png 518.8579
+kitchen/rgb_00781.jpg kitchen/sync_depth_00781.png 518.8579
+kitchen/rgb_00782.jpg kitchen/sync_depth_00782.png 518.8579
+kitchen/rgb_00783.jpg kitchen/sync_depth_00783.png 518.8579
+kitchen/rgb_00784.jpg kitchen/sync_depth_00784.png 518.8579
+kitchen/rgb_00785.jpg kitchen/sync_depth_00785.png 518.8579
+kitchen/rgb_00786.jpg kitchen/sync_depth_00786.png 518.8579
+kitchen/rgb_00799.jpg kitchen/sync_depth_00799.png 518.8579
+kitchen/rgb_00800.jpg kitchen/sync_depth_00800.png 518.8579
+kitchen/rgb_00801.jpg kitchen/sync_depth_00801.png 518.8579
+kitchen/rgb_00802.jpg kitchen/sync_depth_00802.png 518.8579
+kitchen/rgb_00803.jpg kitchen/sync_depth_00803.png 518.8579
+kitchen/rgb_00809.jpg kitchen/sync_depth_00809.png 518.8579
+kitchen/rgb_00810.jpg kitchen/sync_depth_00810.png 518.8579
+kitchen/rgb_00812.jpg kitchen/sync_depth_00812.png 518.8579
+kitchen/rgb_00813.jpg kitchen/sync_depth_00813.png 518.8579
+kitchen/rgb_00820.jpg kitchen/sync_depth_00820.png 518.8579
+kitchen/rgb_00821.jpg kitchen/sync_depth_00821.png 518.8579
+kitchen/rgb_00822.jpg kitchen/sync_depth_00822.png 518.8579
+kitchen/rgb_00832.jpg kitchen/sync_depth_00832.png 518.8579
+kitchen/rgb_00833.jpg kitchen/sync_depth_00833.png 518.8579
+kitchen/rgb_00834.jpg kitchen/sync_depth_00834.png 518.8579
+kitchen/rgb_00835.jpg kitchen/sync_depth_00835.png 518.8579
+kitchen/rgb_00836.jpg kitchen/sync_depth_00836.png 518.8579
+kitchen/rgb_00837.jpg kitchen/sync_depth_00837.png 518.8579
+kitchen/rgb_00838.jpg kitchen/sync_depth_00838.png 518.8579
+kitchen/rgb_00839.jpg kitchen/sync_depth_00839.png 518.8579
+kitchen/rgb_00840.jpg kitchen/sync_depth_00840.png 518.8579
+kitchen/rgb_00841.jpg kitchen/sync_depth_00841.png 518.8579
+kitchen/rgb_00842.jpg kitchen/sync_depth_00842.png 518.8579
+kitchen/rgb_00843.jpg kitchen/sync_depth_00843.png 518.8579
+kitchen/rgb_00845.jpg kitchen/sync_depth_00845.png 518.8579
+kitchen/rgb_00849.jpg kitchen/sync_depth_00849.png 518.8579
+kitchen/rgb_00850.jpg kitchen/sync_depth_00850.png 518.8579
+kitchen/rgb_00851.jpg kitchen/sync_depth_00851.png 518.8579
+kitchen/rgb_00856.jpg kitchen/sync_depth_00856.png 518.8579
+kitchen/rgb_00857.jpg kitchen/sync_depth_00857.png 518.8579
+kitchen/rgb_00858.jpg kitchen/sync_depth_00858.png 518.8579
+kitchen/rgb_00859.jpg kitchen/sync_depth_00859.png 518.8579
+kitchen/rgb_00860.jpg kitchen/sync_depth_00860.png 518.8579
+kitchen/rgb_00861.jpg kitchen/sync_depth_00861.png 518.8579
+kitchen/rgb_00868.jpg kitchen/sync_depth_00868.png 518.8579
+kitchen/rgb_00869.jpg kitchen/sync_depth_00869.png 518.8579
+kitchen/rgb_00870.jpg kitchen/sync_depth_00870.png 518.8579
+kitchen/rgb_00905.jpg kitchen/sync_depth_00905.png 518.8579
+kitchen/rgb_00906.jpg kitchen/sync_depth_00906.png 518.8579
+kitchen/rgb_00907.jpg kitchen/sync_depth_00907.png 518.8579
+living_room/rgb_00152.jpg living_room/sync_depth_00152.png 518.8579
+living_room/rgb_00153.jpg living_room/sync_depth_00153.png 518.8579
+living_room/rgb_00154.jpg living_room/sync_depth_00154.png 518.8579
+living_room/rgb_00166.jpg living_room/sync_depth_00166.png 518.8579
+living_room/rgb_00167.jpg living_room/sync_depth_00167.png 518.8579
+living_room/rgb_00168.jpg living_room/sync_depth_00168.png 518.8579
+living_room/rgb_00206.jpg living_room/sync_depth_00206.png 518.8579
+living_room/rgb_00207.jpg living_room/sync_depth_00207.png 518.8579
+living_room/rgb_00208.jpg living_room/sync_depth_00208.png 518.8579
+living_room/rgb_00209.jpg living_room/sync_depth_00209.png 518.8579
+living_room/rgb_00210.jpg living_room/sync_depth_00210.png 518.8579
+living_room/rgb_00211.jpg living_room/sync_depth_00211.png 518.8579
+living_room/rgb_00263.jpg living_room/sync_depth_00263.png 518.8579
+living_room/rgb_00578.jpg living_room/sync_depth_00578.png 518.8579
+living_room/rgb_00579.jpg living_room/sync_depth_00579.png 518.8579
+living_room/rgb_00580.jpg living_room/sync_depth_00580.png 518.8579
+living_room/rgb_00581.jpg living_room/sync_depth_00581.png 518.8579
+living_room/rgb_00590.jpg living_room/sync_depth_00590.png 518.8579
+living_room/rgb_00591.jpg living_room/sync_depth_00591.png 518.8579
+living_room/rgb_00592.jpg living_room/sync_depth_00592.png 518.8579
+living_room/rgb_00593.jpg living_room/sync_depth_00593.png 518.8579
+living_room/rgb_00602.jpg living_room/sync_depth_00602.png 518.8579
+living_room/rgb_00603.jpg living_room/sync_depth_00603.png 518.8579
+living_room/rgb_00604.jpg living_room/sync_depth_00604.png 518.8579
+living_room/rgb_00605.jpg living_room/sync_depth_00605.png 518.8579
+living_room/rgb_00606.jpg living_room/sync_depth_00606.png 518.8579
+living_room/rgb_01200.jpg living_room/sync_depth_01200.png 518.8579
+living_room/rgb_01201.jpg living_room/sync_depth_01201.png 518.8579
+living_room/rgb_01202.jpg living_room/sync_depth_01202.png 518.8579
+living_room/rgb_01203.jpg living_room/sync_depth_01203.png 518.8579
+living_room/rgb_01204.jpg living_room/sync_depth_01204.png 518.8579
+living_room/rgb_01205.jpg living_room/sync_depth_01205.png 518.8579
+living_room/rgb_01206.jpg living_room/sync_depth_01206.png 518.8579
+living_room/rgb_01207.jpg living_room/sync_depth_01207.png 518.8579
+living_room/rgb_00582.jpg living_room/sync_depth_00582.png 518.8579
+living_room/rgb_01208.jpg living_room/sync_depth_01208.png 518.8579
+living_room/rgb_01247.jpg living_room/sync_depth_01247.png 518.8579
+living_room/rgb_01277.jpg living_room/sync_depth_01277.png 518.8579
+living_room/rgb_01302.jpg living_room/sync_depth_01302.png 518.8579
+living_room/rgb_01209.jpg living_room/sync_depth_01209.png 518.8579
+living_room/rgb_01210.jpg living_room/sync_depth_01210.png 518.8579
+living_room/rgb_01211.jpg living_room/sync_depth_01211.png 518.8579
+living_room/rgb_01215.jpg living_room/sync_depth_01215.png 518.8579
+living_room/rgb_01216.jpg living_room/sync_depth_01216.png 518.8579
+living_room/rgb_01217.jpg living_room/sync_depth_01217.png 518.8579
+living_room/rgb_01218.jpg living_room/sync_depth_01218.png 518.8579
+living_room/rgb_01219.jpg living_room/sync_depth_01219.png 518.8579
+living_room/rgb_01225.jpg living_room/sync_depth_01225.png 518.8579
+living_room/rgb_01226.jpg living_room/sync_depth_01226.png 518.8579
+living_room/rgb_01227.jpg living_room/sync_depth_01227.png 518.8579
+living_room/rgb_01228.jpg living_room/sync_depth_01228.png 518.8579
+living_room/rgb_01229.jpg living_room/sync_depth_01229.png 518.8579
+living_room/rgb_01232.jpg living_room/sync_depth_01232.png 518.8579
+living_room/rgb_01233.jpg living_room/sync_depth_01233.png 518.8579
+living_room/rgb_01234.jpg living_room/sync_depth_01234.png 518.8579
+living_room/rgb_01246.jpg living_room/sync_depth_01246.png 518.8579
+living_room/rgb_01248.jpg living_room/sync_depth_01248.png 518.8579
+living_room/rgb_01249.jpg living_room/sync_depth_01249.png 518.8579
+living_room/rgb_01253.jpg living_room/sync_depth_01253.png 518.8579
+living_room/rgb_01254.jpg living_room/sync_depth_01254.png 518.8579
+living_room/rgb_01255.jpg living_room/sync_depth_01255.png 518.8579
+living_room/rgb_01256.jpg living_room/sync_depth_01256.png 518.8579
+living_room/rgb_01257.jpg living_room/sync_depth_01257.png 518.8579
+living_room/rgb_01258.jpg living_room/sync_depth_01258.png 518.8579
+living_room/rgb_01259.jpg living_room/sync_depth_01259.png 518.8579
+living_room/rgb_01260.jpg living_room/sync_depth_01260.png 518.8579
+living_room/rgb_01261.jpg living_room/sync_depth_01261.png 518.8579
+living_room/rgb_01262.jpg living_room/sync_depth_01262.png 518.8579
+living_room/rgb_01263.jpg living_room/sync_depth_01263.png 518.8579
+living_room/rgb_01264.jpg living_room/sync_depth_01264.png 518.8579
+living_room/rgb_01274.jpg living_room/sync_depth_01274.png 518.8579
+living_room/rgb_01275.jpg living_room/sync_depth_01275.png 518.8579
+living_room/rgb_01276.jpg living_room/sync_depth_01276.png 518.8579
+living_room/rgb_01278.jpg living_room/sync_depth_01278.png 518.8579
+living_room/rgb_01279.jpg living_room/sync_depth_01279.png 518.8579
+living_room/rgb_01284.jpg living_room/sync_depth_01284.png 518.8579
+living_room/rgb_01285.jpg living_room/sync_depth_01285.png 518.8579
+living_room/rgb_01286.jpg living_room/sync_depth_01286.png 518.8579
+living_room/rgb_01287.jpg living_room/sync_depth_01287.png 518.8579
+living_room/rgb_01288.jpg living_room/sync_depth_01288.png 518.8579
+living_room/rgb_01289.jpg living_room/sync_depth_01289.png 518.8579
+living_room/rgb_01290.jpg living_room/sync_depth_01290.png 518.8579
+living_room/rgb_01291.jpg living_room/sync_depth_01291.png 518.8579
+living_room/rgb_01292.jpg living_room/sync_depth_01292.png 518.8579
+living_room/rgb_01293.jpg living_room/sync_depth_01293.png 518.8579
+living_room/rgb_01294.jpg living_room/sync_depth_01294.png 518.8579
+living_room/rgb_01296.jpg living_room/sync_depth_01296.png 518.8579
+living_room/rgb_01297.jpg living_room/sync_depth_01297.png 518.8579
+living_room/rgb_01298.jpg living_room/sync_depth_01298.png 518.8579
+living_room/rgb_01301.jpg living_room/sync_depth_01301.png 518.8579
+living_room/rgb_01303.jpg living_room/sync_depth_01303.png 518.8579
+living_room/rgb_01304.jpg living_room/sync_depth_01304.png 518.8579
+living_room/rgb_01305.jpg living_room/sync_depth_01305.png 518.8579
+living_room/rgb_01306.jpg living_room/sync_depth_01306.png 518.8579
+living_room/rgb_01307.jpg living_room/sync_depth_01307.png 518.8579
+living_room/rgb_01313.jpg living_room/sync_depth_01313.png 518.8579
+living_room/rgb_01314.jpg living_room/sync_depth_01314.png 518.8579
+living_room/rgb_01328.jpg living_room/sync_depth_01328.png 518.8579
+living_room/rgb_01329.jpg living_room/sync_depth_01329.png 518.8579
+living_room/rgb_01330.jpg living_room/sync_depth_01330.png 518.8579
+living_room/rgb_01331.jpg living_room/sync_depth_01331.png 518.8579
+living_room/rgb_01334.jpg living_room/sync_depth_01334.png 518.8579
+living_room/rgb_01335.jpg living_room/sync_depth_01335.png 518.8579
+living_room/rgb_01336.jpg living_room/sync_depth_01336.png 518.8579
+living_room/rgb_01337.jpg living_room/sync_depth_01337.png 518.8579
+living_room/rgb_01338.jpg living_room/sync_depth_01338.png 518.8579
+living_room/rgb_01339.jpg living_room/sync_depth_01339.png 518.8579
+office/rgb_00008.jpg office/sync_depth_00008.png 518.8579
+office/rgb_00013.jpg office/sync_depth_00013.png 518.8579
+office/rgb_00014.jpg office/sync_depth_00014.png 518.8579
+office/rgb_00015.jpg office/sync_depth_00015.png 518.8579
+office/rgb_00016.jpg office/sync_depth_00016.png 518.8579
+office/rgb_00017.jpg office/sync_depth_00017.png 518.8579
+office/rgb_00020.jpg office/sync_depth_00020.png 518.8579
+office/rgb_00027.jpg office/sync_depth_00027.png 518.8579
+office/rgb_00028.jpg office/sync_depth_00028.png 518.8579
+office/rgb_00029.jpg office/sync_depth_00029.png 518.8579
+office/rgb_00030.jpg office/sync_depth_00030.png 518.8579
+office/rgb_00031.jpg office/sync_depth_00031.png 518.8579
+office/rgb_00032.jpg office/sync_depth_00032.png 518.8579
+office/rgb_00033.jpg office/sync_depth_00033.png 518.8579
+office/rgb_00034.jpg office/sync_depth_00034.png 518.8579
+office/rgb_00035.jpg office/sync_depth_00035.png 518.8579
+office/rgb_00036.jpg office/sync_depth_00036.png 518.8579
+office/rgb_00038.jpg office/sync_depth_00038.png 518.8579
+office/rgb_00039.jpg office/sync_depth_00039.png 518.8579
+office/rgb_00040.jpg office/sync_depth_00040.png 518.8579
+office/rgb_00041.jpg office/sync_depth_00041.png 518.8579
+office/rgb_00042.jpg office/sync_depth_00042.png 518.8579
+office/rgb_00270.jpg office/sync_depth_00270.png 518.8579
+office/rgb_00271.jpg office/sync_depth_00271.png 518.8579
+office/rgb_00611.jpg office/sync_depth_00611.png 518.8579
+office/rgb_00612.jpg office/sync_depth_00612.png 518.8579
+office/rgb_00616.jpg office/sync_depth_00616.png 518.8579
+office/rgb_00617.jpg office/sync_depth_00617.png 518.8579
+office/rgb_00618.jpg office/sync_depth_00618.png 518.8579
+office/rgb_00619.jpg office/sync_depth_00619.png 518.8579
+office/rgb_00620.jpg office/sync_depth_00620.png 518.8579
+office/rgb_00632.jpg office/sync_depth_00632.png 518.8579
+office/rgb_00633.jpg office/sync_depth_00633.png 518.8579
+office/rgb_00634.jpg office/sync_depth_00634.png 518.8579
+office/rgb_00635.jpg office/sync_depth_00635.png 518.8579
+office/rgb_00636.jpg office/sync_depth_00636.png 518.8579
+office/rgb_00637.jpg office/sync_depth_00637.png 518.8579
+office/rgb_00037.jpg office/sync_depth_00037.png 518.8579
+office_kitchen/rgb_00410.jpg office_kitchen/sync_depth_00410.png 518.8579
+office_kitchen/rgb_00411.jpg office_kitchen/sync_depth_00411.png 518.8579
+office_kitchen/rgb_00412.jpg office_kitchen/sync_depth_00412.png 518.8579
+office_kitchen/rgb_00413.jpg office_kitchen/sync_depth_00413.png 518.8579
+playroom/rgb_00429.jpg playroom/sync_depth_00429.png 518.8579
+playroom/rgb_00430.jpg playroom/sync_depth_00430.png 518.8579
+playroom/rgb_00431.jpg playroom/sync_depth_00431.png 518.8579
+playroom/rgb_00432.jpg playroom/sync_depth_00432.png 518.8579
+playroom/rgb_00433.jpg playroom/sync_depth_00433.png 518.8579
+playroom/rgb_00434.jpg playroom/sync_depth_00434.png 518.8579
+playroom/rgb_00440.jpg playroom/sync_depth_00440.png 518.8579
+playroom/rgb_00441.jpg playroom/sync_depth_00441.png 518.8579
+playroom/rgb_00442.jpg playroom/sync_depth_00442.png 518.8579
+playroom/rgb_00443.jpg playroom/sync_depth_00443.png 518.8579
+playroom/rgb_00444.jpg playroom/sync_depth_00444.png 518.8579
+playroom/rgb_00445.jpg playroom/sync_depth_00445.png 518.8579
+playroom/rgb_00446.jpg playroom/sync_depth_00446.png 518.8579
+playroom/rgb_00447.jpg playroom/sync_depth_00447.png 518.8579
+reception_room/rgb_00461.jpg reception_room/sync_depth_00461.png 518.8579
+reception_room/rgb_00462.jpg reception_room/sync_depth_00462.png 518.8579
+reception_room/rgb_00463.jpg reception_room/sync_depth_00463.png 518.8579
+reception_room/rgb_00464.jpg reception_room/sync_depth_00464.png 518.8579
+reception_room/rgb_00465.jpg reception_room/sync_depth_00465.png 518.8579
+study/rgb_00468.jpg study/sync_depth_00468.png 518.8579
+study/rgb_00469.jpg study/sync_depth_00469.png 518.8579
+study/rgb_00470.jpg study/sync_depth_00470.png 518.8579
+study/rgb_00471.jpg study/sync_depth_00471.png 518.8579
+study/rgb_00472.jpg study/sync_depth_00472.png 518.8579
+study/rgb_00473.jpg study/sync_depth_00473.png 518.8579
+study/rgb_00474.jpg study/sync_depth_00474.png 518.8579
+study/rgb_00475.jpg study/sync_depth_00475.png 518.8579
+study/rgb_00476.jpg study/sync_depth_00476.png 518.8579
+study/rgb_00643.jpg study/sync_depth_00643.png 518.8579
+study/rgb_00644.jpg study/sync_depth_00644.png 518.8579
+study_room/rgb_00272.jpg study_room/sync_depth_00272.png 518.8579
+study_room/rgb_00278.jpg study_room/sync_depth_00278.png 518.8579

ZoeDepth/ui/app.py

@@ -0,0 +1,66 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import gradio as gr
+import torch
+
+from .gradio_depth_pred import create_demo as create_depth_pred_demo
+from .gradio_im_to_3d import create_demo as create_im_to_3d_demo
+from .gradio_pano_to_3d import create_demo as create_pano_to_3d_demo
+
+
+css = """
+#img-display-container {
+    max-height: 50vh;
+    }
+#img-display-input {
+    max-height: 40vh;
+    }
+#img-display-output {
+    max-height: 40vh;
+    }
+    
+"""
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+model = torch.hub.load('isl-org/ZoeDepth', "ZoeD_N", pretrained=True).to(DEVICE).eval()
+
+title = "# ZoeDepth"
+description = """Official demo for **ZoeDepth: Zero-shot Transfer by Combining Relative and Metric Depth**.
+
+ZoeDepth is a deep learning model for metric depth estimation from a single image.
+
+Please refer to our [paper](https://arxiv.org/abs/2302.12288) or [github](https://github.com/isl-org/ZoeDepth) for more details."""
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown(title)
+    gr.Markdown(description)
+    with gr.Tab("Depth Prediction"):
+        create_depth_pred_demo(model)
+    with gr.Tab("Image to 3D"):
+        create_im_to_3d_demo(model)
+    with gr.Tab("360 Panorama to 3D"):
+        create_pano_to_3d_demo(model)
+
+if __name__ == '__main__':
+    demo.queue().launch()

ZoeDepth/ui/gradio_depth_pred.py

@@ -0,0 +1,52 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import gradio as gr
+from zoedepth.utils.misc import colorize
+from PIL import Image
+import tempfile
+
+def predict_depth(model, image):
+    depth = model.infer_pil(image)
+    return depth
+
+def create_demo(model):
+    gr.Markdown("### Depth Prediction demo")
+    with gr.Row():
+        input_image = gr.Image(label="Input Image", type='pil', elem_id='img-display-input').style(height="auto")
+        depth_image = gr.Image(label="Depth Map", elem_id='img-display-output')
+    raw_file = gr.File(label="16-bit raw depth, multiplier:256")
+    submit = gr.Button("Submit")
+
+    def on_submit(image):
+        depth = predict_depth(model, image)
+        colored_depth = colorize(depth, cmap='gray_r')
+        tmp = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
+        raw_depth = Image.fromarray((depth*256).astype('uint16'))
+        raw_depth.save(tmp.name)
+        return [colored_depth, tmp.name]
+    
+    submit.click(on_submit, inputs=[input_image], outputs=[depth_image, raw_file])
+    # examples = gr.Examples(examples=["examples/person_1.jpeg", "examples/person_2.jpeg", "examples/person-leaves.png", "examples/living-room.jpeg"],
+    #                        inputs=[input_image])

ZoeDepth/ui/gradio_im_to_3d.py

@@ -0,0 +1,93 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import gradio as gr
+import numpy as np
+import trimesh
+from zoedepth.utils.geometry import depth_to_points, create_triangles
+from functools import partial
+import tempfile
+
+
+def depth_edges_mask(depth):
+    """Returns a mask of edges in the depth map.
+    Args:
+    depth: 2D numpy array of shape (H, W) with dtype float32.
+    Returns:
+    mask: 2D numpy array of shape (H, W) with dtype bool.
+    """
+    # Compute the x and y gradients of the depth map.
+    depth_dx, depth_dy = np.gradient(depth)
+    # Compute the gradient magnitude.
+    depth_grad = np.sqrt(depth_dx ** 2 + depth_dy ** 2)
+    # Compute the edge mask.
+    mask = depth_grad > 0.05
+    return mask
+
+
+def predict_depth(model, image):
+    depth = model.infer_pil(image)
+    return depth
+
+def get_mesh(model, image, keep_edges=False):
+    image.thumbnail((1024,1024))  # limit the size of the input image
+    depth = predict_depth(model, image)
+    pts3d = depth_to_points(depth[None])
+    pts3d = pts3d.reshape(-1, 3)
+
+    # Create a trimesh mesh from the points
+    # Each pixel is connected to its 4 neighbors
+    # colors are the RGB values of the image
+
+    verts = pts3d.reshape(-1, 3)
+    image = np.array(image)
+    if keep_edges:
+        triangles = create_triangles(image.shape[0], image.shape[1])
+    else:
+        triangles = create_triangles(image.shape[0], image.shape[1], mask=~depth_edges_mask(depth))
+    colors = image.reshape(-1, 3)
+    mesh = trimesh.Trimesh(vertices=verts, faces=triangles, vertex_colors=colors)
+
+    # Save as glb
+    glb_file = tempfile.NamedTemporaryFile(suffix='.glb', delete=False)
+    glb_path = glb_file.name
+    mesh.export(glb_path)
+    return glb_path
+
+def create_demo(model):
+
+    gr.Markdown("### Image to 3D mesh")
+    gr.Markdown("Convert a single 2D image to a 3D mesh")
+
+    with gr.Row():
+        image = gr.Image(label="Input Image", type='pil')
+        result = gr.Model3D(label="3d mesh reconstruction", clear_color=[
+                                                 1.0, 1.0, 1.0, 1.0])
+    
+    checkbox = gr.Checkbox(label="Keep occlusion edges", value=False)
+    submit = gr.Button("Submit")
+    submit.click(partial(get_mesh, model), inputs=[image, checkbox], outputs=[result])
+    # examples = gr.Examples(examples=["examples/aerial_beach.jpeg", "examples/mountains.jpeg", "examples/person_1.jpeg", "examples/ancient-carved.jpeg"],
+    #                         inputs=[image])
+

ZoeDepth/ui/gradio_pano_to_3d.py

@@ -0,0 +1,120 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import gradio as gr
+import numpy as np
+import trimesh
+from zoedepth.utils.geometry import create_triangles
+from functools import partial
+import tempfile
+
+def depth_edges_mask(depth):
+    """Returns a mask of edges in the depth map.
+    Args:
+    depth: 2D numpy array of shape (H, W) with dtype float32.
+    Returns:
+    mask: 2D numpy array of shape (H, W) with dtype bool.
+    """
+    # Compute the x and y gradients of the depth map.
+    depth_dx, depth_dy = np.gradient(depth)
+    # Compute the gradient magnitude.
+    depth_grad = np.sqrt(depth_dx ** 2 + depth_dy ** 2)
+    # Compute the edge mask.
+    mask = depth_grad > 0.05
+    return mask
+
+
+def pano_depth_to_world_points(depth):
+    """
+    360 depth to world points
+    given 2D depth is an equirectangular projection of a spherical image
+    Treat depth as radius
+
+    longitude : -pi to pi
+    latitude : -pi/2 to pi/2
+    """
+
+    # Convert depth to radius
+    radius = depth.flatten()
+
+    lon = np.linspace(-np.pi, np.pi, depth.shape[1])
+    lat = np.linspace(-np.pi/2, np.pi/2, depth.shape[0])
+
+    lon, lat = np.meshgrid(lon, lat)
+    lon = lon.flatten()
+    lat = lat.flatten()
+
+    # Convert to cartesian coordinates
+    x = radius * np.cos(lat) * np.cos(lon)
+    y = radius * np.cos(lat) * np.sin(lon)
+    z = radius * np.sin(lat)
+
+    pts3d = np.stack([x, y, z], axis=1)
+
+    return pts3d
+
+
+def predict_depth(model, image):
+    depth = model.infer_pil(image)
+    return depth
+
+def get_mesh(model, image, keep_edges=False):
+    image.thumbnail((1024,1024))  # limit the size of the image
+    depth = predict_depth(model, image)
+    pts3d = pano_depth_to_world_points(depth)
+
+    # Create a trimesh mesh from the points
+    # Each pixel is connected to its 4 neighbors
+    # colors are the RGB values of the image
+
+    verts = pts3d.reshape(-1, 3)
+    image = np.array(image)
+    if keep_edges:
+        triangles = create_triangles(image.shape[0], image.shape[1])
+    else:
+        triangles = create_triangles(image.shape[0], image.shape[1], mask=~depth_edges_mask(depth))
+    colors = image.reshape(-1, 3)
+    mesh = trimesh.Trimesh(vertices=verts, faces=triangles, vertex_colors=colors)
+
+    # Save as glb
+    glb_file = tempfile.NamedTemporaryFile(suffix='.glb', delete=False)
+    glb_path = glb_file.name
+    mesh.export(glb_path)
+    return glb_path
+
+def create_demo(model):
+    gr.Markdown("### Panorama to 3D mesh")
+    gr.Markdown("Convert a 360 spherical panorama to a 3D mesh")
+    gr.Markdown("ZoeDepth was not trained on panoramic images. It doesn't know anything about panoramas or spherical projection. Here, we just treat the estimated depth as radius and some projection errors are expected. Nonetheless, ZoeDepth still works surprisingly well on 360 reconstruction.")
+
+    with gr.Row():
+        input_image = gr.Image(label="Input Image", type='pil')
+        result = gr.Model3D(label="3d mesh reconstruction", clear_color=[
+                                                 1.0, 1.0, 1.0, 1.0])
+        
+    checkbox = gr.Checkbox(label="Keep occlusion edges", value=True)
+    submit = gr.Button("Submit")
+    submit.click(partial(get_mesh, model), inputs=[input_image, checkbox], outputs=[result])
+    # examples = gr.Examples(examples=["examples/pano_1.jpeg", "examples/pano_2.jpeg", "examples/pano_3.jpeg"],
+    #                         inputs=[input_image])

ZoeDepth/ui/ui_requirements.txt

@@ -0,0 +1,2 @@
+gradio
+trimesh==3.9.42

ZoeDepth/zoedepth/data/__init__.py

@@ -0,0 +1,24 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+

ZoeDepth/zoedepth/data/data_mono.py

@@ -0,0 +1,573 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+# This file is partly inspired from BTS (https://github.com/cleinc/bts/blob/master/pytorch/bts_dataloader.py); author: Jin Han Lee
+
+import itertools
+import os
+import random
+
+import numpy as np
+import cv2
+import torch
+import torch.nn as nn
+import torch.utils.data.distributed
+from zoedepth.utils.easydict import EasyDict as edict
+from PIL import Image, ImageOps
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+
+from zoedepth.utils.config import change_dataset
+
+from .ddad import get_ddad_loader
+from .diml_indoor_test import get_diml_indoor_loader
+from .diml_outdoor_test import get_diml_outdoor_loader
+from .diode import get_diode_loader
+from .hypersim import get_hypersim_loader
+from .ibims import get_ibims_loader
+from .sun_rgbd_loader import get_sunrgbd_loader
+from .vkitti import get_vkitti_loader
+from .vkitti2 import get_vkitti2_loader
+
+from .preprocess import CropParams, get_white_border, get_black_border
+
+
+def _is_pil_image(img):
+    return isinstance(img, Image.Image)
+
+
+def _is_numpy_image(img):
+    return isinstance(img, np.ndarray) and (img.ndim in {2, 3})
+
+
+def preprocessing_transforms(mode, **kwargs):
+    return transforms.Compose([
+        ToTensor(mode=mode, **kwargs)
+    ])
+
+
+class DepthDataLoader(object):
+    def __init__(self, config, mode, device='cpu', transform=None, **kwargs):
+        """
+        Data loader for depth datasets
+
+        Args:
+            config (dict): Config dictionary. Refer to utils/config.py
+            mode (str): "train" or "online_eval"
+            device (str, optional): Device to load the data on. Defaults to 'cpu'.
+            transform (torchvision.transforms, optional): Transform to apply to the data. Defaults to None.
+        """
+
+        self.config = config
+
+        if config.dataset == 'ibims':
+            self.data = get_ibims_loader(config, batch_size=1, num_workers=1)
+            return
+
+        if config.dataset == 'sunrgbd':
+            self.data = get_sunrgbd_loader(
+                data_dir_root=config.sunrgbd_root, batch_size=1, num_workers=1)
+            return
+
+        if config.dataset == 'diml_indoor':
+            self.data = get_diml_indoor_loader(
+                data_dir_root=config.diml_indoor_root, batch_size=1, num_workers=1)
+            return
+
+        if config.dataset == 'diml_outdoor':
+            self.data = get_diml_outdoor_loader(
+                data_dir_root=config.diml_outdoor_root, batch_size=1, num_workers=1)
+            return
+
+        if "diode" in config.dataset:
+            self.data = get_diode_loader(
+                config[config.dataset+"_root"], batch_size=1, num_workers=1)
+            return
+
+        if config.dataset == 'hypersim_test':
+            self.data = get_hypersim_loader(
+                config.hypersim_test_root, batch_size=1, num_workers=1)
+            return
+
+        if config.dataset == 'vkitti':
+            self.data = get_vkitti_loader(
+                config.vkitti_root, batch_size=1, num_workers=1)
+            return
+
+        if config.dataset == 'vkitti2':
+            self.data = get_vkitti2_loader(
+                config.vkitti2_root, batch_size=1, num_workers=1)
+            return
+
+        if config.dataset == 'ddad':
+            self.data = get_ddad_loader(config.ddad_root, resize_shape=(
+                352, 1216), batch_size=1, num_workers=1)
+            return
+
+        img_size = self.config.get("img_size", None)
+        img_size = img_size if self.config.get(
+            "do_input_resize", False) else None
+
+        if transform is None:
+            transform = preprocessing_transforms(mode, size=img_size)
+
+        if mode == 'train':
+
+            Dataset = DataLoadPreprocess
+            self.training_samples = Dataset(
+                config, mode, transform=transform, device=device)
+
+            if config.distributed:
+                self.train_sampler = torch.utils.data.distributed.DistributedSampler(
+                    self.training_samples)
+            else:
+                self.train_sampler = None
+
+            self.data = DataLoader(self.training_samples,
+                                   batch_size=config.batch_size,
+                                   shuffle=(self.train_sampler is None),
+                                   num_workers=config.workers,
+                                   pin_memory=True,
+                                   persistent_workers=True,
+                                #    prefetch_factor=2,
+                                   sampler=self.train_sampler)
+
+        elif mode == 'online_eval':
+            self.testing_samples = DataLoadPreprocess(
+                config, mode, transform=transform)
+            if config.distributed:  # redundant. here only for readability and to be more explicit
+                # Give whole test set to all processes (and report evaluation only on one) regardless
+                self.eval_sampler = None
+            else:
+                self.eval_sampler = None
+            self.data = DataLoader(self.testing_samples, 1,
+                                   shuffle=kwargs.get("shuffle_test", False),
+                                   num_workers=1,
+                                   pin_memory=False,
+                                   sampler=self.eval_sampler)
+
+        elif mode == 'test':
+            self.testing_samples = DataLoadPreprocess(
+                config, mode, transform=transform)
+            self.data = DataLoader(self.testing_samples,
+                                   1, shuffle=False, num_workers=1)
+
+        else:
+            print(
+                'mode should be one of \'train, test, online_eval\'. Got {}'.format(mode))
+
+
+def repetitive_roundrobin(*iterables):
+    """
+    cycles through iterables but sample wise
+    first yield first sample from first iterable then first sample from second iterable and so on
+    then second sample from first iterable then second sample from second iterable and so on
+
+    If one iterable is shorter than the others, it is repeated until all iterables are exhausted
+    repetitive_roundrobin('ABC', 'D', 'EF') --> A D E B D F C D E
+    """
+    # Repetitive roundrobin
+    iterables_ = [iter(it) for it in iterables]
+    exhausted = [False] * len(iterables)
+    while not all(exhausted):
+        for i, it in enumerate(iterables_):
+            try:
+                yield next(it)
+            except StopIteration:
+                exhausted[i] = True
+                iterables_[i] = itertools.cycle(iterables[i])
+                # First elements may get repeated if one iterable is shorter than the others
+                yield next(iterables_[i])
+
+
+class RepetitiveRoundRobinDataLoader(object):
+    def __init__(self, *dataloaders):
+        self.dataloaders = dataloaders
+
+    def __iter__(self):
+        return repetitive_roundrobin(*self.dataloaders)
+
+    def __len__(self):
+        # First samples get repeated, thats why the plus one
+        return len(self.dataloaders) * (max(len(dl) for dl in self.dataloaders) + 1)
+
+
+class MixedNYUKITTI(object):
+    def __init__(self, config, mode, device='cpu', **kwargs):
+        config = edict(config)
+        config.workers = config.workers // 2
+        self.config = config
+        nyu_conf = change_dataset(edict(config), 'nyu')
+        kitti_conf = change_dataset(edict(config), 'kitti')
+
+        # make nyu default for testing
+        self.config = config = nyu_conf
+        img_size = self.config.get("img_size", None)
+        img_size = img_size if self.config.get(
+            "do_input_resize", False) else None
+        if mode == 'train':
+            nyu_loader = DepthDataLoader(
+                nyu_conf, mode, device=device, transform=preprocessing_transforms(mode, size=img_size)).data
+            kitti_loader = DepthDataLoader(
+                kitti_conf, mode, device=device, transform=preprocessing_transforms(mode, size=img_size)).data
+            # It has been changed to repetitive roundrobin
+            self.data = RepetitiveRoundRobinDataLoader(
+                nyu_loader, kitti_loader)
+        else:
+            self.data = DepthDataLoader(nyu_conf, mode, device=device).data
+
+
+def remove_leading_slash(s):
+    if s[0] == '/' or s[0] == '\\':
+        return s[1:]
+    return s
+
+
+class CachedReader:
+    def __init__(self, shared_dict=None):
+        if shared_dict:
+            self._cache = shared_dict
+        else:
+            self._cache = {}
+
+    def open(self, fpath):
+        im = self._cache.get(fpath, None)
+        if im is None:
+            im = self._cache[fpath] = Image.open(fpath)
+        return im
+
+
+class ImReader:
+    def __init__(self):
+        pass
+
+    # @cache
+    def open(self, fpath):
+        return Image.open(fpath)
+
+
+class DataLoadPreprocess(Dataset):
+    def __init__(self, config, mode, transform=None, is_for_online_eval=False, **kwargs):
+        self.config = config
+        if mode == 'online_eval':
+            with open(config.filenames_file_eval, 'r') as f:
+                self.filenames = f.readlines()
+        else:
+            with open(config.filenames_file, 'r') as f:
+                self.filenames = f.readlines()
+
+        self.mode = mode
+        self.transform = transform
+        self.to_tensor = ToTensor(mode)
+        self.is_for_online_eval = is_for_online_eval
+        if config.use_shared_dict:
+            self.reader = CachedReader(config.shared_dict)
+        else:
+            self.reader = ImReader()
+
+    def postprocess(self, sample):
+        return sample
+
+    def __getitem__(self, idx):
+        sample_path = self.filenames[idx]
+        focal = float(sample_path.split()[2])
+        sample = {}
+
+        if self.mode == 'train':
+            if self.config.dataset == 'kitti' and self.config.use_right and random.random() > 0.5:
+                image_path = os.path.join(
+                    self.config.data_path, remove_leading_slash(sample_path.split()[3]))
+                depth_path = os.path.join(
+                    self.config.gt_path, remove_leading_slash(sample_path.split()[4]))
+            else:
+                image_path = os.path.join(
+                    self.config.data_path, remove_leading_slash(sample_path.split()[0]))
+                depth_path = os.path.join(
+                    self.config.gt_path, remove_leading_slash(sample_path.split()[1]))
+
+            image = self.reader.open(image_path)
+            depth_gt = self.reader.open(depth_path)
+            w, h = image.size
+
+            if self.config.do_kb_crop:
+                height = image.height
+                width = image.width
+                top_margin = int(height - 352)
+                left_margin = int((width - 1216) / 2)
+                depth_gt = depth_gt.crop(
+                    (left_margin, top_margin, left_margin + 1216, top_margin + 352))
+                image = image.crop(
+                    (left_margin, top_margin, left_margin + 1216, top_margin + 352))
+
+            # Avoid blank boundaries due to pixel registration?
+            # Train images have white border. Test images have black border.
+            if self.config.dataset == 'nyu' and self.config.avoid_boundary:
+                # print("Avoiding Blank Boundaries!")
+                # We just crop and pad again with reflect padding to original size
+                # original_size = image.size
+                crop_params = get_white_border(np.array(image, dtype=np.uint8))
+                image = image.crop((crop_params.left, crop_params.top, crop_params.right, crop_params.bottom))
+                depth_gt = depth_gt.crop((crop_params.left, crop_params.top, crop_params.right, crop_params.bottom))
+
+                # Use reflect padding to fill the blank
+                image = np.array(image)
+                image = np.pad(image, ((crop_params.top, h - crop_params.bottom), (crop_params.left, w - crop_params.right), (0, 0)), mode='reflect')
+                image = Image.fromarray(image)
+
+                depth_gt = np.array(depth_gt)
+                depth_gt = np.pad(depth_gt, ((crop_params.top, h - crop_params.bottom), (crop_params.left, w - crop_params.right)), 'constant', constant_values=0)
+                depth_gt = Image.fromarray(depth_gt)
+
+
+            if self.config.do_random_rotate and (self.config.aug):
+                random_angle = (random.random() - 0.5) * 2 * self.config.degree
+                image = self.rotate_image(image, random_angle)
+                depth_gt = self.rotate_image(
+                    depth_gt, random_angle, flag=Image.NEAREST)
+
+            image = np.asarray(image, dtype=np.float32) / 255.0
+            depth_gt = np.asarray(depth_gt, dtype=np.float32)
+            depth_gt = np.expand_dims(depth_gt, axis=2)
+
+            if self.config.dataset == 'nyu':
+                depth_gt = depth_gt / 1000.0
+            else:
+                depth_gt = depth_gt / 256.0
+
+            if self.config.aug and (self.config.random_crop):
+                image, depth_gt = self.random_crop(
+                    image, depth_gt, self.config.input_height, self.config.input_width)
+            
+            if self.config.aug and self.config.random_translate:
+                # print("Random Translation!")
+                image, depth_gt = self.random_translate(image, depth_gt, self.config.max_translation)
+
+            image, depth_gt = self.train_preprocess(image, depth_gt)
+            mask = np.logical_and(depth_gt > self.config.min_depth,
+                                  depth_gt < self.config.max_depth).squeeze()[None, ...]
+            sample = {'image': image, 'depth': depth_gt, 'focal': focal,
+                      'mask': mask, **sample}
+
+        else:
+            if self.mode == 'online_eval':
+                data_path = self.config.data_path_eval
+            else:
+                data_path = self.config.data_path
+
+            image_path = os.path.join(
+                data_path, remove_leading_slash(sample_path.split()[0]))
+            image = np.asarray(self.reader.open(image_path),
+                               dtype=np.float32) / 255.0
+
+            if self.mode == 'online_eval':
+                gt_path = self.config.gt_path_eval
+                depth_path = os.path.join(
+                    gt_path, remove_leading_slash(sample_path.split()[1]))
+                has_valid_depth = False
+                try:
+                    depth_gt = self.reader.open(depth_path)
+                    has_valid_depth = True
+                except IOError:
+                    depth_gt = False
+                    # print('Missing gt for {}'.format(image_path))
+
+                if has_valid_depth:
+                    depth_gt = np.asarray(depth_gt, dtype=np.float32)
+                    depth_gt = np.expand_dims(depth_gt, axis=2)
+                    if self.config.dataset == 'nyu':
+                        depth_gt = depth_gt / 1000.0
+                    else:
+                        depth_gt = depth_gt / 256.0
+
+                    mask = np.logical_and(
+                        depth_gt >= self.config.min_depth, depth_gt <= self.config.max_depth).squeeze()[None, ...]
+                else:
+                    mask = False
+
+            if self.config.do_kb_crop:
+                height = image.shape[0]
+                width = image.shape[1]
+                top_margin = int(height - 352)
+                left_margin = int((width - 1216) / 2)
+                image = image[top_margin:top_margin + 352,
+                              left_margin:left_margin + 1216, :]
+                if self.mode == 'online_eval' and has_valid_depth:
+                    depth_gt = depth_gt[top_margin:top_margin +
+                                        352, left_margin:left_margin + 1216, :]
+
+            if self.mode == 'online_eval':
+                sample = {'image': image, 'depth': depth_gt, 'focal': focal, 'has_valid_depth': has_valid_depth,
+                          'image_path': sample_path.split()[0], 'depth_path': sample_path.split()[1],
+                          'mask': mask}
+            else:
+                sample = {'image': image, 'focal': focal}
+
+        if (self.mode == 'train') or ('has_valid_depth' in sample and sample['has_valid_depth']):
+            mask = np.logical_and(depth_gt > self.config.min_depth,
+                                  depth_gt < self.config.max_depth).squeeze()[None, ...]
+            sample['mask'] = mask
+
+        if self.transform:
+            sample = self.transform(sample)
+
+        sample = self.postprocess(sample)
+        sample['dataset'] = self.config.dataset
+        sample = {**sample, 'image_path': sample_path.split()[0], 'depth_path': sample_path.split()[1]}
+
+        return sample
+
+    def rotate_image(self, image, angle, flag=Image.BILINEAR):
+        result = image.rotate(angle, resample=flag)
+        return result
+
+    def random_crop(self, img, depth, height, width):
+        assert img.shape[0] >= height
+        assert img.shape[1] >= width
+        assert img.shape[0] == depth.shape[0]
+        assert img.shape[1] == depth.shape[1]
+        x = random.randint(0, img.shape[1] - width)
+        y = random.randint(0, img.shape[0] - height)
+        img = img[y:y + height, x:x + width, :]
+        depth = depth[y:y + height, x:x + width, :]
+
+        return img, depth
+    
+    def random_translate(self, img, depth, max_t=20):
+        assert img.shape[0] == depth.shape[0]
+        assert img.shape[1] == depth.shape[1]
+        p = self.config.translate_prob
+        do_translate = random.random()
+        if do_translate > p:
+            return img, depth
+        x = random.randint(-max_t, max_t)
+        y = random.randint(-max_t, max_t)
+        M = np.float32([[1, 0, x], [0, 1, y]])
+        # print(img.shape, depth.shape)
+        img = cv2.warpAffine(img, M, (img.shape[1], img.shape[0]))
+        depth = cv2.warpAffine(depth, M, (depth.shape[1], depth.shape[0]))
+        depth = depth.squeeze()[..., None]  # add channel dim back. Affine warp removes it
+        # print("after", img.shape, depth.shape)
+        return img, depth
+
+    def train_preprocess(self, image, depth_gt):
+        if self.config.aug:
+            # Random flipping
+            do_flip = random.random()
+            if do_flip > 0.5:
+                image = (image[:, ::-1, :]).copy()
+                depth_gt = (depth_gt[:, ::-1, :]).copy()
+
+            # Random gamma, brightness, color augmentation
+            do_augment = random.random()
+            if do_augment > 0.5:
+                image = self.augment_image(image)
+
+        return image, depth_gt
+
+    def augment_image(self, image):
+        # gamma augmentation
+        gamma = random.uniform(0.9, 1.1)
+        image_aug = image ** gamma
+
+        # brightness augmentation
+        if self.config.dataset == 'nyu':
+            brightness = random.uniform(0.75, 1.25)
+        else:
+            brightness = random.uniform(0.9, 1.1)
+        image_aug = image_aug * brightness
+
+        # color augmentation
+        colors = np.random.uniform(0.9, 1.1, size=3)
+        white = np.ones((image.shape[0], image.shape[1]))
+        color_image = np.stack([white * colors[i] for i in range(3)], axis=2)
+        image_aug *= color_image
+        image_aug = np.clip(image_aug, 0, 1)
+
+        return image_aug
+
+    def __len__(self):
+        return len(self.filenames)
+
+
+class ToTensor(object):
+    def __init__(self, mode, do_normalize=False, size=None):
+        self.mode = mode
+        self.normalize = transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) if do_normalize else nn.Identity()
+        self.size = size
+        if size is not None:
+            self.resize = transforms.Resize(size=size)
+        else:
+            self.resize = nn.Identity()
+
+    def __call__(self, sample):
+        image, focal = sample['image'], sample['focal']
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        image = self.resize(image)
+
+        if self.mode == 'test':
+            return {'image': image, 'focal': focal}
+
+        depth = sample['depth']
+        if self.mode == 'train':
+            depth = self.to_tensor(depth)
+            return {**sample, 'image': image, 'depth': depth, 'focal': focal}
+        else:
+            has_valid_depth = sample['has_valid_depth']
+            image = self.resize(image)
+            return {**sample, 'image': image, 'depth': depth, 'focal': focal, 'has_valid_depth': has_valid_depth,
+                    'image_path': sample['image_path'], 'depth_path': sample['depth_path']}
+
+    def to_tensor(self, pic):
+        if not (_is_pil_image(pic) or _is_numpy_image(pic)):
+            raise TypeError(
+                'pic should be PIL Image or ndarray. Got {}'.format(type(pic)))
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img

ZoeDepth/zoedepth/data/ddad.py

@@ -0,0 +1,117 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import os
+
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+
+
+class ToTensor(object):
+    def __init__(self, resize_shape):
+        # self.normalize = transforms.Normalize(
+        #     mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        self.normalize = lambda x : x
+        self.resize = transforms.Resize(resize_shape)
+
+    def __call__(self, sample):
+        image, depth = sample['image'], sample['depth']
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        depth = self.to_tensor(depth)
+
+        image = self.resize(image)
+
+        return {'image': image, 'depth': depth, 'dataset': "ddad"}
+
+    def to_tensor(self, pic):
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        #         # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img
+
+
+class DDAD(Dataset):
+    def __init__(self, data_dir_root, resize_shape):
+        import glob
+
+        # image paths are of the form <data_dir_root>/{outleft, depthmap}/*.png
+        self.image_files = glob.glob(os.path.join(data_dir_root, '*.png'))
+        self.depth_files = [r.replace("_rgb.png", "_depth.npy")
+                            for r in self.image_files]
+        self.transform = ToTensor(resize_shape)
+
+    def __getitem__(self, idx):
+
+        image_path = self.image_files[idx]
+        depth_path = self.depth_files[idx]
+
+        image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0
+        depth = np.load(depth_path)  # meters
+
+        # depth[depth > 8] = -1
+        depth = depth[..., None]
+
+        sample = dict(image=image, depth=depth)
+        sample = self.transform(sample)
+
+        if idx == 0:
+            print(sample["image"].shape)
+
+        return sample
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+def get_ddad_loader(data_dir_root, resize_shape, batch_size=1, **kwargs):
+    dataset = DDAD(data_dir_root, resize_shape)
+    return DataLoader(dataset, batch_size, **kwargs)

ZoeDepth/zoedepth/data/diml_indoor_test.py

@@ -0,0 +1,125 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import os
+
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+
+
+class ToTensor(object):
+    def __init__(self):
+        # self.normalize = transforms.Normalize(
+        #     mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        self.normalize = lambda x : x
+        self.resize = transforms.Resize((480, 640))
+
+    def __call__(self, sample):
+        image, depth = sample['image'], sample['depth']
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        depth = self.to_tensor(depth)
+
+        image = self.resize(image)
+
+        return {'image': image, 'depth': depth, 'dataset': "diml_indoor"}
+
+    def to_tensor(self, pic):
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        #         # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img
+
+
+class DIML_Indoor(Dataset):
+    def __init__(self, data_dir_root):
+        import glob
+
+        # image paths are of the form <data_dir_root>/{HR, LR}/<scene>/{color, depth_filled}/*.png
+        self.image_files = glob.glob(os.path.join(
+            data_dir_root, "LR", '*', 'color', '*.png'))
+        self.depth_files = [r.replace("color", "depth_filled").replace(
+            "_c.png", "_depth_filled.png") for r in self.image_files]
+        self.transform = ToTensor()
+
+    def __getitem__(self, idx):
+        image_path = self.image_files[idx]
+        depth_path = self.depth_files[idx]
+
+        image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0
+        depth = np.asarray(Image.open(depth_path),
+                           dtype='uint16') / 1000.0  # mm to meters
+
+        # print(np.shape(image))
+        # print(np.shape(depth))
+
+        # depth[depth > 8] = -1
+        depth = depth[..., None]
+
+        sample = dict(image=image, depth=depth)
+
+        # return sample
+        sample = self.transform(sample)
+
+        if idx == 0:
+            print(sample["image"].shape)
+
+        return sample
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+def get_diml_indoor_loader(data_dir_root, batch_size=1, **kwargs):
+    dataset = DIML_Indoor(data_dir_root)
+    return DataLoader(dataset, batch_size, **kwargs)
+
+# get_diml_indoor_loader(data_dir_root="datasets/diml/indoor/test/HR")
+# get_diml_indoor_loader(data_dir_root="datasets/diml/indoor/test/LR")

ZoeDepth/zoedepth/data/diml_outdoor_test.py

@@ -0,0 +1,114 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import os
+
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+
+
+class ToTensor(object):
+    def __init__(self):
+        # self.normalize = transforms.Normalize(
+        #     mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        self.normalize = lambda x : x
+
+    def __call__(self, sample):
+        image, depth = sample['image'], sample['depth']
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        depth = self.to_tensor(depth)
+
+        return {'image': image, 'depth': depth, 'dataset': "diml_outdoor"}
+
+    def to_tensor(self, pic):
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        #         # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img
+
+
+class DIML_Outdoor(Dataset):
+    def __init__(self, data_dir_root):
+        import glob
+
+        # image paths are of the form <data_dir_root>/{outleft, depthmap}/*.png
+        self.image_files = glob.glob(os.path.join(
+            data_dir_root, "*", 'outleft', '*.png'))
+        self.depth_files = [r.replace("outleft", "depthmap")
+                            for r in self.image_files]
+        self.transform = ToTensor()
+
+    def __getitem__(self, idx):
+        image_path = self.image_files[idx]
+        depth_path = self.depth_files[idx]
+
+        image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0
+        depth = np.asarray(Image.open(depth_path),
+                           dtype='uint16') / 1000.0  # mm to meters
+
+        # depth[depth > 8] = -1
+        depth = depth[..., None]
+
+        sample = dict(image=image, depth=depth, dataset="diml_outdoor")
+
+        # return sample
+        return self.transform(sample)
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+def get_diml_outdoor_loader(data_dir_root, batch_size=1, **kwargs):
+    dataset = DIML_Outdoor(data_dir_root)
+    return DataLoader(dataset, batch_size, **kwargs)
+
+# get_diml_outdoor_loader(data_dir_root="datasets/diml/outdoor/test/HR")
+# get_diml_outdoor_loader(data_dir_root="datasets/diml/outdoor/test/LR")

ZoeDepth/zoedepth/data/diode.py

@@ -0,0 +1,125 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import os
+
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+
+
+class ToTensor(object):
+    def __init__(self):
+        # self.normalize = transforms.Normalize(
+        #     mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        self.normalize = lambda x : x
+        self.resize = transforms.Resize(480)
+
+    def __call__(self, sample):
+        image, depth = sample['image'], sample['depth']
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        depth = self.to_tensor(depth)
+
+        image = self.resize(image)
+
+        return {'image': image, 'depth': depth, 'dataset': "diode"}
+
+    def to_tensor(self, pic):
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        #         # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img
+
+
+class DIODE(Dataset):
+    def __init__(self, data_dir_root):
+        import glob
+
+        # image paths are of the form <data_dir_root>/scene_#/scan_#/*.png
+        self.image_files = glob.glob(
+            os.path.join(data_dir_root, '*', '*', '*.png'))
+        self.depth_files = [r.replace(".png", "_depth.npy")
+                            for r in self.image_files]
+        self.depth_mask_files = [
+            r.replace(".png", "_depth_mask.npy") for r in self.image_files]
+        self.transform = ToTensor()
+
+    def __getitem__(self, idx):
+        image_path = self.image_files[idx]
+        depth_path = self.depth_files[idx]
+        depth_mask_path = self.depth_mask_files[idx]
+
+        image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0
+        depth = np.load(depth_path)  # in meters
+        valid = np.load(depth_mask_path)  # binary
+
+        # depth[depth > 8] = -1
+        # depth = depth[..., None]
+
+        sample = dict(image=image, depth=depth, valid=valid)
+
+        # return sample
+        sample = self.transform(sample)
+
+        if idx == 0:
+            print(sample["image"].shape)
+
+        return sample
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+def get_diode_loader(data_dir_root, batch_size=1, **kwargs):
+    dataset = DIODE(data_dir_root)
+    return DataLoader(dataset, batch_size, **kwargs)
+
+# get_diode_loader(data_dir_root="datasets/diode/val/outdoor")

ZoeDepth/zoedepth/data/hypersim.py

@@ -0,0 +1,138 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import glob
+import os
+
+import h5py
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+
+
+def hypersim_distance_to_depth(npyDistance):
+    intWidth, intHeight, fltFocal = 1024, 768, 886.81
+
+    npyImageplaneX = np.linspace((-0.5 * intWidth) + 0.5, (0.5 * intWidth) - 0.5, intWidth).reshape(
+        1, intWidth).repeat(intHeight, 0).astype(np.float32)[:, :, None]
+    npyImageplaneY = np.linspace((-0.5 * intHeight) + 0.5, (0.5 * intHeight) - 0.5,
+                                 intHeight).reshape(intHeight, 1).repeat(intWidth, 1).astype(np.float32)[:, :, None]
+    npyImageplaneZ = np.full([intHeight, intWidth, 1], fltFocal, np.float32)
+    npyImageplane = np.concatenate(
+        [npyImageplaneX, npyImageplaneY, npyImageplaneZ], 2)
+
+    npyDepth = npyDistance / np.linalg.norm(npyImageplane, 2, 2) * fltFocal
+    return npyDepth
+
+
+class ToTensor(object):
+    def __init__(self):
+        # self.normalize = transforms.Normalize(
+        #     mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        self.normalize = lambda x: x
+        self.resize = transforms.Resize((480, 640))
+
+    def __call__(self, sample):
+        image, depth = sample['image'], sample['depth']
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        depth = self.to_tensor(depth)
+
+        image = self.resize(image)
+
+        return {'image': image, 'depth': depth, 'dataset': "hypersim"}
+
+    def to_tensor(self, pic):
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        #         # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img
+
+
+class HyperSim(Dataset):
+    def __init__(self, data_dir_root):
+        # image paths are of the form <data_dir_root>/<scene>/images/scene_cam_#_final_preview/*.tonemap.jpg
+        # depth paths are of the form <data_dir_root>/<scene>/images/scene_cam_#_final_preview/*.depth_meters.hdf5
+        self.image_files = glob.glob(os.path.join(
+            data_dir_root, '*', 'images', 'scene_cam_*_final_preview', '*.tonemap.jpg'))
+        self.depth_files = [r.replace("_final_preview", "_geometry_hdf5").replace(
+            ".tonemap.jpg", ".depth_meters.hdf5") for r in self.image_files]
+        self.transform = ToTensor()
+
+    def __getitem__(self, idx):
+        image_path = self.image_files[idx]
+        depth_path = self.depth_files[idx]
+
+        image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0
+
+        # depth from hdf5
+        depth_fd = h5py.File(depth_path, "r")
+        # in meters (Euclidean distance)
+        distance_meters = np.array(depth_fd['dataset'])
+        depth = hypersim_distance_to_depth(
+            distance_meters)  # in meters (planar depth)
+
+        # depth[depth > 8] = -1
+        depth = depth[..., None]
+
+        sample = dict(image=image, depth=depth)
+        sample = self.transform(sample)
+
+        if idx == 0:
+            print(sample["image"].shape)
+
+        return sample
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+def get_hypersim_loader(data_dir_root, batch_size=1, **kwargs):
+    dataset = HyperSim(data_dir_root)
+    return DataLoader(dataset, batch_size, **kwargs)

ZoeDepth/zoedepth/data/ibims.py

@@ -0,0 +1,81 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import os
+
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms as T
+
+
+class iBims(Dataset):
+    def __init__(self, config):
+        root_folder = config.ibims_root
+        with open(os.path.join(root_folder, "imagelist.txt"), 'r') as f:
+            imglist = f.read().split()
+
+        samples = []
+        for basename in imglist:
+            img_path = os.path.join(root_folder, 'rgb', basename + ".png")
+            depth_path = os.path.join(root_folder, 'depth', basename + ".png")
+            valid_mask_path = os.path.join(
+                root_folder, 'mask_invalid', basename+".png")
+            transp_mask_path = os.path.join(
+                root_folder, 'mask_transp', basename+".png")
+
+            samples.append(
+                (img_path, depth_path, valid_mask_path, transp_mask_path))
+
+        self.samples = samples
+        # self.normalize = T.Normalize(
+        #     mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        self.normalize = lambda x : x
+
+    def __getitem__(self, idx):
+        img_path, depth_path, valid_mask_path, transp_mask_path = self.samples[idx]
+
+        img = np.asarray(Image.open(img_path), dtype=np.float32) / 255.0
+        depth = np.asarray(Image.open(depth_path),
+                           dtype=np.uint16).astype('float')*50.0/65535
+
+        mask_valid = np.asarray(Image.open(valid_mask_path))
+        mask_transp = np.asarray(Image.open(transp_mask_path))
+
+        # depth = depth * mask_valid * mask_transp
+        depth = np.where(mask_valid * mask_transp, depth, -1)
+
+        img = torch.from_numpy(img).permute(2, 0, 1)
+        img = self.normalize(img)
+        depth = torch.from_numpy(depth).unsqueeze(0)
+        return dict(image=img, depth=depth, image_path=img_path, depth_path=depth_path, dataset='ibims')
+
+    def __len__(self):
+        return len(self.samples)
+
+
+def get_ibims_loader(config, batch_size=1, **kwargs):
+    dataloader = DataLoader(iBims(config), batch_size=batch_size, **kwargs)
+    return dataloader

ZoeDepth/zoedepth/data/preprocess.py

@@ -0,0 +1,154 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import numpy as np
+from dataclasses import dataclass
+from typing import Tuple, List
+
+# dataclass to store the crop parameters
+@dataclass
+class CropParams:
+    top: int
+    bottom: int
+    left: int
+    right: int
+
+
+
+def get_border_params(rgb_image, tolerance=0.1, cut_off=20, value=0, level_diff_threshold=5, channel_axis=-1, min_border=5) -> CropParams:
+    gray_image = np.mean(rgb_image, axis=channel_axis)
+    h, w = gray_image.shape
+
+
+    def num_value_pixels(arr):
+        return np.sum(np.abs(arr - value) < level_diff_threshold)
+
+    def is_above_tolerance(arr, total_pixels):
+        return (num_value_pixels(arr) / total_pixels) > tolerance
+
+    # Crop top border until number of value pixels become below tolerance
+    top = min_border
+    while is_above_tolerance(gray_image[top, :], w) and top < h-1:
+        top += 1
+        if top > cut_off:
+            break
+
+    # Crop bottom border until number of value pixels become below tolerance
+    bottom = h - min_border
+    while is_above_tolerance(gray_image[bottom, :], w) and bottom > 0:
+        bottom -= 1
+        if h - bottom > cut_off:
+            break
+
+    # Crop left border until number of value pixels become below tolerance
+    left = min_border
+    while is_above_tolerance(gray_image[:, left], h) and left < w-1:
+        left += 1
+        if left > cut_off:
+            break
+
+    # Crop right border until number of value pixels become below tolerance
+    right = w - min_border
+    while is_above_tolerance(gray_image[:, right], h) and right > 0:
+        right -= 1
+        if w - right > cut_off:
+            break
+        
+
+    return CropParams(top, bottom, left, right)
+
+
+def get_white_border(rgb_image, value=255, **kwargs) -> CropParams:
+    """Crops the white border of the RGB.
+
+    Args:
+        rgb: RGB image, shape (H, W, 3).
+    Returns:
+        Crop parameters.
+    """
+    if value == 255:
+        # assert range of values in rgb image is [0, 255]
+        assert np.max(rgb_image) <= 255 and np.min(rgb_image) >= 0, "RGB image values are not in range [0, 255]."
+        assert rgb_image.max() > 1, "RGB image values are not in range [0, 255]."
+    elif value == 1:
+        # assert range of values in rgb image is [0, 1]
+        assert np.max(rgb_image) <= 1 and np.min(rgb_image) >= 0, "RGB image values are not in range [0, 1]."
+
+    return get_border_params(rgb_image, value=value, **kwargs)
+
+def get_black_border(rgb_image, **kwargs) -> CropParams:
+    """Crops the black border of the RGB.
+
+    Args:
+        rgb: RGB image, shape (H, W, 3).
+
+    Returns:
+        Crop parameters.
+    """
+
+    return get_border_params(rgb_image, value=0, **kwargs)
+
+def crop_image(image: np.ndarray, crop_params: CropParams) -> np.ndarray:
+    """Crops the image according to the crop parameters.
+
+    Args:
+        image: RGB or depth image, shape (H, W, 3) or (H, W).
+        crop_params: Crop parameters.
+
+    Returns:
+        Cropped image.
+    """
+    return image[crop_params.top:crop_params.bottom, crop_params.left:crop_params.right]
+
+def crop_images(*images: np.ndarray, crop_params: CropParams) -> Tuple[np.ndarray]:
+    """Crops the images according to the crop parameters.
+
+    Args:
+        images: RGB or depth images, shape (H, W, 3) or (H, W).
+        crop_params: Crop parameters.
+
+    Returns:
+        Cropped images.
+    """
+    return tuple(crop_image(image, crop_params) for image in images)
+
+def crop_black_or_white_border(rgb_image, *other_images: np.ndarray, tolerance=0.1, cut_off=20, level_diff_threshold=5) -> Tuple[np.ndarray]:
+    """Crops the white and black border of the RGB and depth images.
+
+    Args:
+        rgb: RGB image, shape (H, W, 3). This image is used to determine the border.
+        other_images: The other images to crop according to the border of the RGB image.
+    Returns:
+        Cropped RGB and other images.
+    """
+    # crop black border
+    crop_params = get_black_border(rgb_image, tolerance=tolerance, cut_off=cut_off, level_diff_threshold=level_diff_threshold)
+    cropped_images = crop_images(rgb_image, *other_images, crop_params=crop_params)
+
+    # crop white border
+    crop_params = get_white_border(cropped_images[0], tolerance=tolerance, cut_off=cut_off, level_diff_threshold=level_diff_threshold)
+    cropped_images = crop_images(*cropped_images, crop_params=crop_params)
+
+    return cropped_images
+    

ZoeDepth/zoedepth/data/sun_rgbd_loader.py

@@ -0,0 +1,106 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import os
+
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+
+
+class ToTensor(object):
+    def __init__(self):
+        # self.normalize = transforms.Normalize(
+        #     mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        self.normalize = lambda x : x
+
+    def __call__(self, sample):
+        image, depth = sample['image'], sample['depth']
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        depth = self.to_tensor(depth)
+
+        return {'image': image, 'depth': depth, 'dataset': "sunrgbd"}
+
+    def to_tensor(self, pic):
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        #         # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img
+
+
+class SunRGBD(Dataset):
+    def __init__(self, data_dir_root):
+        # test_file_dirs = loadmat(train_test_file)['alltest'].squeeze()
+        # all_test = [t[0].replace("/n/fs/sun3d/data/", "") for t in test_file_dirs]
+        # self.all_test = [os.path.join(data_dir_root, t) for t in all_test]
+        import glob
+        self.image_files = glob.glob(
+            os.path.join(data_dir_root, 'rgb', 'rgb', '*'))
+        self.depth_files = [
+            r.replace("rgb/rgb", "gt/gt").replace("jpg", "png") for r in self.image_files]
+        self.transform = ToTensor()
+
+    def __getitem__(self, idx):
+        image_path = self.image_files[idx]
+        depth_path = self.depth_files[idx]
+
+        image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0
+        depth = np.asarray(Image.open(depth_path), dtype='uint16') / 1000.0
+        depth[depth > 8] = -1
+        depth = depth[..., None]
+        return self.transform(dict(image=image, depth=depth))
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+def get_sunrgbd_loader(data_dir_root, batch_size=1, **kwargs):
+    dataset = SunRGBD(data_dir_root)
+    return DataLoader(dataset, batch_size, **kwargs)

ZoeDepth/zoedepth/data/transforms.py

@@ -0,0 +1,481 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import math
+import random
+
+import cv2
+import numpy as np
+
+
+class RandomFliplr(object):
+    """Horizontal flip of the sample with given probability.
+    """
+
+    def __init__(self, probability=0.5):
+        """Init.
+
+        Args:
+            probability (float, optional): Flip probability. Defaults to 0.5.
+        """
+        self.__probability = probability
+
+    def __call__(self, sample):
+        prob = random.random()
+
+        if prob < self.__probability:
+            for k, v in sample.items():
+                if len(v.shape) >= 2:
+                    sample[k] = np.fliplr(v).copy()
+
+        return sample
+
+
+def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA):
+    """Rezise the sample to ensure the given size. Keeps aspect ratio.
+
+    Args:
+        sample (dict): sample
+        size (tuple): image size
+
+    Returns:
+        tuple: new size
+    """
+    shape = list(sample["disparity"].shape)
+
+    if shape[0] >= size[0] and shape[1] >= size[1]:
+        return sample
+
+    scale = [0, 0]
+    scale[0] = size[0] / shape[0]
+    scale[1] = size[1] / shape[1]
+
+    scale = max(scale)
+
+    shape[0] = math.ceil(scale * shape[0])
+    shape[1] = math.ceil(scale * shape[1])
+
+    # resize
+    sample["image"] = cv2.resize(
+        sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method
+    )
+
+    sample["disparity"] = cv2.resize(
+        sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST
+    )
+    sample["mask"] = cv2.resize(
+        sample["mask"].astype(np.float32),
+        tuple(shape[::-1]),
+        interpolation=cv2.INTER_NEAREST,
+    )
+    sample["mask"] = sample["mask"].astype(bool)
+
+    return tuple(shape)
+
+
+class RandomCrop(object):
+    """Get a random crop of the sample with the given size (width, height).
+    """
+
+    def __init__(
+        self,
+        width,
+        height,
+        resize_if_needed=False,
+        image_interpolation_method=cv2.INTER_AREA,
+    ):
+        """Init.
+
+        Args:
+            width (int): output width
+            height (int): output height
+            resize_if_needed (bool, optional): If True, sample might be upsampled to ensure
+                that a crop of size (width, height) is possbile. Defaults to False.
+        """
+        self.__size = (height, width)
+        self.__resize_if_needed = resize_if_needed
+        self.__image_interpolation_method = image_interpolation_method
+
+    def __call__(self, sample):
+
+        shape = sample["disparity"].shape
+
+        if self.__size[0] > shape[0] or self.__size[1] > shape[1]:
+            if self.__resize_if_needed:
+                shape = apply_min_size(
+                    sample, self.__size, self.__image_interpolation_method
+                )
+            else:
+                raise Exception(
+                    "Output size {} bigger than input size {}.".format(
+                        self.__size, shape
+                    )
+                )
+
+        offset = (
+            np.random.randint(shape[0] - self.__size[0] + 1),
+            np.random.randint(shape[1] - self.__size[1] + 1),
+        )
+
+        for k, v in sample.items():
+            if k == "code" or k == "basis":
+                continue
+
+            if len(sample[k].shape) >= 2:
+                sample[k] = v[
+                    offset[0]: offset[0] + self.__size[0],
+                    offset[1]: offset[1] + self.__size[1],
+                ]
+
+        return sample
+
+
+class Resize(object):
+    """Resize sample to given size (width, height).
+    """
+
+    def __init__(
+        self,
+        width,
+        height,
+        resize_target=True,
+        keep_aspect_ratio=False,
+        ensure_multiple_of=1,
+        resize_method="lower_bound",
+        image_interpolation_method=cv2.INTER_AREA,
+        letter_box=False,
+    ):
+        """Init.
+
+        Args:
+            width (int): desired output width
+            height (int): desired output height
+            resize_target (bool, optional):
+                True: Resize the full sample (image, mask, target).
+                False: Resize image only.
+                Defaults to True.
+            keep_aspect_ratio (bool, optional):
+                True: Keep the aspect ratio of the input sample.
+                Output sample might not have the given width and height, and
+                resize behaviour depends on the parameter 'resize_method'.
+                Defaults to False.
+            ensure_multiple_of (int, optional):
+                Output width and height is constrained to be multiple of this parameter.
+                Defaults to 1.
+            resize_method (str, optional):
+                "lower_bound": Output will be at least as large as the given size.
+                "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.)
+                "minimal": Scale as least as possible.  (Output size might be smaller than given size.)
+                Defaults to "lower_bound".
+        """
+        self.__width = width
+        self.__height = height
+
+        self.__resize_target = resize_target
+        self.__keep_aspect_ratio = keep_aspect_ratio
+        self.__multiple_of = ensure_multiple_of
+        self.__resize_method = resize_method
+        self.__image_interpolation_method = image_interpolation_method
+        self.__letter_box = letter_box
+
+    def constrain_to_multiple_of(self, x, min_val=0, max_val=None):
+        y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int)
+
+        if max_val is not None and y > max_val:
+            y = (np.floor(x / self.__multiple_of)
+                 * self.__multiple_of).astype(int)
+
+        if y < min_val:
+            y = (np.ceil(x / self.__multiple_of)
+                 * self.__multiple_of).astype(int)
+
+        return y
+
+    def get_size(self, width, height):
+        # determine new height and width
+        scale_height = self.__height / height
+        scale_width = self.__width / width
+
+        if self.__keep_aspect_ratio:
+            if self.__resize_method == "lower_bound":
+                # scale such that output size is lower bound
+                if scale_width > scale_height:
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            elif self.__resize_method == "upper_bound":
+                # scale such that output size is upper bound
+                if scale_width < scale_height:
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            elif self.__resize_method == "minimal":
+                # scale as least as possbile
+                if abs(1 - scale_width) < abs(1 - scale_height):
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            else:
+                raise ValueError(
+                    f"resize_method {self.__resize_method} not implemented"
+                )
+
+        if self.__resize_method == "lower_bound":
+            new_height = self.constrain_to_multiple_of(
+                scale_height * height, min_val=self.__height
+            )
+            new_width = self.constrain_to_multiple_of(
+                scale_width * width, min_val=self.__width
+            )
+        elif self.__resize_method == "upper_bound":
+            new_height = self.constrain_to_multiple_of(
+                scale_height * height, max_val=self.__height
+            )
+            new_width = self.constrain_to_multiple_of(
+                scale_width * width, max_val=self.__width
+            )
+        elif self.__resize_method == "minimal":
+            new_height = self.constrain_to_multiple_of(scale_height * height)
+            new_width = self.constrain_to_multiple_of(scale_width * width)
+        else:
+            raise ValueError(
+                f"resize_method {self.__resize_method} not implemented")
+
+        return (new_width, new_height)
+
+    def make_letter_box(self, sample):
+        top = bottom = (self.__height - sample.shape[0]) // 2
+        left = right = (self.__width - sample.shape[1]) // 2
+        sample = cv2.copyMakeBorder(
+            sample, top, bottom, left, right, cv2.BORDER_CONSTANT, None, 0)
+        return sample
+
+    def __call__(self, sample):
+        width, height = self.get_size(
+            sample["image"].shape[1], sample["image"].shape[0]
+        )
+
+        # resize sample
+        sample["image"] = cv2.resize(
+            sample["image"],
+            (width, height),
+            interpolation=self.__image_interpolation_method,
+        )
+
+        if self.__letter_box:
+            sample["image"] = self.make_letter_box(sample["image"])
+
+        if self.__resize_target:
+            if "disparity" in sample:
+                sample["disparity"] = cv2.resize(
+                    sample["disparity"],
+                    (width, height),
+                    interpolation=cv2.INTER_NEAREST,
+                )
+
+                if self.__letter_box:
+                    sample["disparity"] = self.make_letter_box(
+                        sample["disparity"])
+
+            if "depth" in sample:
+                sample["depth"] = cv2.resize(
+                    sample["depth"], (width,
+                                      height), interpolation=cv2.INTER_NEAREST
+                )
+
+                if self.__letter_box:
+                    sample["depth"] = self.make_letter_box(sample["depth"])
+
+            sample["mask"] = cv2.resize(
+                sample["mask"].astype(np.float32),
+                (width, height),
+                interpolation=cv2.INTER_NEAREST,
+            )
+
+            if self.__letter_box:
+                sample["mask"] = self.make_letter_box(sample["mask"])
+
+            sample["mask"] = sample["mask"].astype(bool)
+
+        return sample
+
+
+class ResizeFixed(object):
+    def __init__(self, size):
+        self.__size = size
+
+    def __call__(self, sample):
+        sample["image"] = cv2.resize(
+            sample["image"], self.__size[::-1], interpolation=cv2.INTER_LINEAR
+        )
+
+        sample["disparity"] = cv2.resize(
+            sample["disparity"], self.__size[::-
+                                             1], interpolation=cv2.INTER_NEAREST
+        )
+
+        sample["mask"] = cv2.resize(
+            sample["mask"].astype(np.float32),
+            self.__size[::-1],
+            interpolation=cv2.INTER_NEAREST,
+        )
+        sample["mask"] = sample["mask"].astype(bool)
+
+        return sample
+
+
+class Rescale(object):
+    """Rescale target values to the interval [0, max_val].
+    If input is constant, values are set to max_val / 2.
+    """
+
+    def __init__(self, max_val=1.0, use_mask=True):
+        """Init.
+
+        Args:
+            max_val (float, optional): Max output value. Defaults to 1.0.
+            use_mask (bool, optional): Only operate on valid pixels (mask == True). Defaults to True.
+        """
+        self.__max_val = max_val
+        self.__use_mask = use_mask
+
+    def __call__(self, sample):
+        disp = sample["disparity"]
+
+        if self.__use_mask:
+            mask = sample["mask"]
+        else:
+            mask = np.ones_like(disp, dtype=np.bool)
+
+        if np.sum(mask) == 0:
+            return sample
+
+        min_val = np.min(disp[mask])
+        max_val = np.max(disp[mask])
+
+        if max_val > min_val:
+            sample["disparity"][mask] = (
+                (disp[mask] - min_val) / (max_val - min_val) * self.__max_val
+            )
+        else:
+            sample["disparity"][mask] = np.ones_like(
+                disp[mask]) * self.__max_val / 2.0
+
+        return sample
+
+
+# mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
+class NormalizeImage(object):
+    """Normlize image by given mean and std.
+    """
+
+    def __init__(self, mean, std):
+        self.__mean = mean
+        self.__std = std
+
+    def __call__(self, sample):
+        sample["image"] = (sample["image"] - self.__mean) / self.__std
+
+        return sample
+
+
+class DepthToDisparity(object):
+    """Convert depth to disparity. Removes depth from sample.
+    """
+
+    def __init__(self, eps=1e-4):
+        self.__eps = eps
+
+    def __call__(self, sample):
+        assert "depth" in sample
+
+        sample["mask"][sample["depth"] < self.__eps] = False
+
+        sample["disparity"] = np.zeros_like(sample["depth"])
+        sample["disparity"][sample["depth"] >= self.__eps] = (
+            1.0 / sample["depth"][sample["depth"] >= self.__eps]
+        )
+
+        del sample["depth"]
+
+        return sample
+
+
+class DisparityToDepth(object):
+    """Convert disparity to depth. Removes disparity from sample.
+    """
+
+    def __init__(self, eps=1e-4):
+        self.__eps = eps
+
+    def __call__(self, sample):
+        assert "disparity" in sample
+
+        disp = np.abs(sample["disparity"])
+        sample["mask"][disp < self.__eps] = False
+
+        # print(sample["disparity"])
+        # print(sample["mask"].sum())
+        # exit()
+
+        sample["depth"] = np.zeros_like(disp)
+        sample["depth"][disp >= self.__eps] = (
+            1.0 / disp[disp >= self.__eps]
+        )
+
+        del sample["disparity"]
+
+        return sample
+
+
+class PrepareForNet(object):
+    """Prepare sample for usage as network input.
+    """
+
+    def __init__(self):
+        pass
+
+    def __call__(self, sample):
+        image = np.transpose(sample["image"], (2, 0, 1))
+        sample["image"] = np.ascontiguousarray(image).astype(np.float32)
+
+        if "mask" in sample:
+            sample["mask"] = sample["mask"].astype(np.float32)
+            sample["mask"] = np.ascontiguousarray(sample["mask"])
+
+        if "disparity" in sample:
+            disparity = sample["disparity"].astype(np.float32)
+            sample["disparity"] = np.ascontiguousarray(disparity)
+
+        if "depth" in sample:
+            depth = sample["depth"].astype(np.float32)
+            sample["depth"] = np.ascontiguousarray(depth)
+
+        return sample

ZoeDepth/zoedepth/data/vkitti.py

@@ -0,0 +1,151 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import torch
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms
+import os
+
+from PIL import Image
+import numpy as np
+import cv2
+
+
+class ToTensor(object):
+    def __init__(self):
+        self.normalize = transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        # self.resize = transforms.Resize((375, 1242))
+
+    def __call__(self, sample):
+        image, depth = sample['image'], sample['depth']
+
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        depth = self.to_tensor(depth)
+
+        # image = self.resize(image)
+
+        return {'image': image, 'depth': depth, 'dataset': "vkitti"}
+
+    def to_tensor(self, pic):
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        #         # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img
+
+
+class VKITTI(Dataset):
+    def __init__(self, data_dir_root, do_kb_crop=True):
+        import glob
+        # image paths are of the form <data_dir_root>/{HR, LR}/<scene>/{color, depth_filled}/*.png
+        self.image_files = glob.glob(os.path.join(
+            data_dir_root, "test_color", '*.png'))
+        self.depth_files = [r.replace("test_color", "test_depth")
+                            for r in self.image_files]
+        self.do_kb_crop = True
+        self.transform = ToTensor()
+
+    def __getitem__(self, idx):
+        image_path = self.image_files[idx]
+        depth_path = self.depth_files[idx]
+
+        image = Image.open(image_path)
+        depth = Image.open(depth_path)
+        depth = cv2.imread(depth_path, cv2.IMREAD_ANYCOLOR |
+                           cv2.IMREAD_ANYDEPTH)
+        print("dpeth min max", depth.min(), depth.max())
+
+        # print(np.shape(image))
+        # print(np.shape(depth))
+
+        # depth[depth > 8] = -1
+
+        if self.do_kb_crop and False:
+            height = image.height
+            width = image.width
+            top_margin = int(height - 352)
+            left_margin = int((width - 1216) / 2)
+            depth = depth.crop(
+                (left_margin, top_margin, left_margin + 1216, top_margin + 352))
+            image = image.crop(
+                (left_margin, top_margin, left_margin + 1216, top_margin + 352))
+            # uv = uv[:, top_margin:top_margin + 352, left_margin:left_margin + 1216]
+
+        image = np.asarray(image, dtype=np.float32) / 255.0
+        # depth = np.asarray(depth, dtype=np.uint16) /1.
+        depth = depth[..., None]
+        sample = dict(image=image, depth=depth)
+
+        # return sample
+        sample = self.transform(sample)
+
+        if idx == 0:
+            print(sample["image"].shape)
+
+        return sample
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+def get_vkitti_loader(data_dir_root, batch_size=1, **kwargs):
+    dataset = VKITTI(data_dir_root)
+    return DataLoader(dataset, batch_size, **kwargs)
+
+
+if __name__ == "__main__":
+    loader = get_vkitti_loader(
+        data_dir_root="/home/bhatsf/shortcuts/datasets/vkitti_test")
+    print("Total files", len(loader.dataset))
+    for i, sample in enumerate(loader):
+        print(sample["image"].shape)
+        print(sample["depth"].shape)
+        print(sample["dataset"])
+        print(sample['depth'].min(), sample['depth'].max())
+        if i > 5:
+            break

ZoeDepth/zoedepth/data/vkitti2.py

@@ -0,0 +1,187 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import os
+
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import DataLoader, Dataset
+from torchvision import transforms
+
+
+class ToTensor(object):
+    def __init__(self):
+        # self.normalize = transforms.Normalize(
+        #     mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        self.normalize = lambda x: x
+        # self.resize = transforms.Resize((375, 1242))
+
+    def __call__(self, sample):
+        image, depth = sample['image'], sample['depth']
+
+        image = self.to_tensor(image)
+        image = self.normalize(image)
+        depth = self.to_tensor(depth)
+
+        # image = self.resize(image)
+
+        return {'image': image, 'depth': depth, 'dataset': "vkitti"}
+
+    def to_tensor(self, pic):
+
+        if isinstance(pic, np.ndarray):
+            img = torch.from_numpy(pic.transpose((2, 0, 1)))
+            return img
+
+        #         # handle PIL Image
+        if pic.mode == 'I':
+            img = torch.from_numpy(np.array(pic, np.int32, copy=False))
+        elif pic.mode == 'I;16':
+            img = torch.from_numpy(np.array(pic, np.int16, copy=False))
+        else:
+            img = torch.ByteTensor(
+                torch.ByteStorage.from_buffer(pic.tobytes()))
+        # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK
+        if pic.mode == 'YCbCr':
+            nchannel = 3
+        elif pic.mode == 'I;16':
+            nchannel = 1
+        else:
+            nchannel = len(pic.mode)
+        img = img.view(pic.size[1], pic.size[0], nchannel)
+
+        img = img.transpose(0, 1).transpose(0, 2).contiguous()
+        if isinstance(img, torch.ByteTensor):
+            return img.float()
+        else:
+            return img
+
+
+class VKITTI2(Dataset):
+    def __init__(self, data_dir_root, do_kb_crop=True, split="test"):
+        import glob
+
+        # image paths are of the form <data_dir_root>/rgb/<scene>/<variant>/frames/<rgb,depth>/Camera<0,1>/rgb_{}.jpg
+        self.image_files = glob.glob(os.path.join(
+            data_dir_root, "rgb", "**", "frames", "rgb", "Camera_0", '*.jpg'), recursive=True)
+        self.depth_files = [r.replace("/rgb/", "/depth/").replace(
+            "rgb_", "depth_").replace(".jpg", ".png") for r in self.image_files]
+        self.do_kb_crop = True
+        self.transform = ToTensor()
+
+        # If train test split is not created, then create one.
+        # Split is such that 8% of the frames from each scene are used for testing.
+        if not os.path.exists(os.path.join(data_dir_root, "train.txt")):
+            import random
+            scenes = set([os.path.basename(os.path.dirname(
+                os.path.dirname(os.path.dirname(f)))) for f in self.image_files])
+            train_files = []
+            test_files = []
+            for scene in scenes:
+                scene_files = [f for f in self.image_files if os.path.basename(
+                    os.path.dirname(os.path.dirname(os.path.dirname(f)))) == scene]
+                random.shuffle(scene_files)
+                train_files.extend(scene_files[:int(len(scene_files) * 0.92)])
+                test_files.extend(scene_files[int(len(scene_files) * 0.92):])
+            with open(os.path.join(data_dir_root, "train.txt"), "w") as f:
+                f.write("\n".join(train_files))
+            with open(os.path.join(data_dir_root, "test.txt"), "w") as f:
+                f.write("\n".join(test_files))
+
+        if split == "train":
+            with open(os.path.join(data_dir_root, "train.txt"), "r") as f:
+                self.image_files = f.read().splitlines()
+            self.depth_files = [r.replace("/rgb/", "/depth/").replace(
+                "rgb_", "depth_").replace(".jpg", ".png") for r in self.image_files]
+        elif split == "test":
+            with open(os.path.join(data_dir_root, "test.txt"), "r") as f:
+                self.image_files = f.read().splitlines()
+            self.depth_files = [r.replace("/rgb/", "/depth/").replace(
+                "rgb_", "depth_").replace(".jpg", ".png") for r in self.image_files]
+
+    def __getitem__(self, idx):
+        image_path = self.image_files[idx]
+        depth_path = self.depth_files[idx]
+
+        image = Image.open(image_path)
+        # depth = Image.open(depth_path)
+        depth = cv2.imread(depth_path, cv2.IMREAD_ANYCOLOR |
+                           cv2.IMREAD_ANYDEPTH) / 100.0  # cm to m
+        depth = Image.fromarray(depth)
+        # print("dpeth min max", depth.min(), depth.max())
+
+        # print(np.shape(image))
+        # print(np.shape(depth))
+
+        if self.do_kb_crop:
+            if idx == 0:
+                print("Using KB input crop")
+            height = image.height
+            width = image.width
+            top_margin = int(height - 352)
+            left_margin = int((width - 1216) / 2)
+            depth = depth.crop(
+                (left_margin, top_margin, left_margin + 1216, top_margin + 352))
+            image = image.crop(
+                (left_margin, top_margin, left_margin + 1216, top_margin + 352))
+            # uv = uv[:, top_margin:top_margin + 352, left_margin:left_margin + 1216]
+
+        image = np.asarray(image, dtype=np.float32) / 255.0
+        # depth = np.asarray(depth, dtype=np.uint16) /1.
+        depth = np.asarray(depth, dtype=np.float32) / 1.
+        depth[depth > 80] = -1
+
+        depth = depth[..., None]
+        sample = dict(image=image, depth=depth)
+
+        # return sample
+        sample = self.transform(sample)
+
+        if idx == 0:
+            print(sample["image"].shape)
+
+        return sample
+
+    def __len__(self):
+        return len(self.image_files)
+
+
+def get_vkitti2_loader(data_dir_root, batch_size=1, **kwargs):
+    dataset = VKITTI2(data_dir_root)
+    return DataLoader(dataset, batch_size, **kwargs)
+
+
+if __name__ == "__main__":
+    loader = get_vkitti2_loader(
+        data_dir_root="/home/bhatsf/shortcuts/datasets/vkitti2")
+    print("Total files", len(loader.dataset))
+    for i, sample in enumerate(loader):
+        print(sample["image"].shape)
+        print(sample["depth"].shape)
+        print(sample["dataset"])
+        print(sample['depth'].min(), sample['depth'].max())
+        if i > 5:
+            break

ZoeDepth/zoedepth/models/__init__.py

@@ -0,0 +1,24 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+

ZoeDepth/zoedepth/models/base_models/__init__.py

@@ -0,0 +1,24 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+

ZoeDepth/zoedepth/models/base_models/midas.py

@@ -0,0 +1,377 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import torch
+import torch.nn as nn
+import numpy as np
+from torchvision.transforms import Normalize
+
+
+def denormalize(x):
+    """Reverses the imagenet normalization applied to the input.
+
+    Args:
+        x (torch.Tensor - shape(N,3,H,W)): input tensor
+
+    Returns:
+        torch.Tensor - shape(N,3,H,W): Denormalized input
+    """
+    mean = torch.Tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1).to(x.device)
+    std = torch.Tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1).to(x.device)
+    return x * std + mean
+
+def get_activation(name, bank):
+    def hook(model, input, output):
+        bank[name] = output
+    return hook
+
+
+class Resize(object):
+    """Resize sample to given size (width, height).
+    """
+
+    def __init__(
+        self,
+        width,
+        height,
+        resize_target=True,
+        keep_aspect_ratio=False,
+        ensure_multiple_of=1,
+        resize_method="lower_bound",
+    ):
+        """Init.
+        Args:
+            width (int): desired output width
+            height (int): desired output height
+            resize_target (bool, optional):
+                True: Resize the full sample (image, mask, target).
+                False: Resize image only.
+                Defaults to True.
+            keep_aspect_ratio (bool, optional):
+                True: Keep the aspect ratio of the input sample.
+                Output sample might not have the given width and height, and
+                resize behaviour depends on the parameter 'resize_method'.
+                Defaults to False.
+            ensure_multiple_of (int, optional):
+                Output width and height is constrained to be multiple of this parameter.
+                Defaults to 1.
+            resize_method (str, optional):
+                "lower_bound": Output will be at least as large as the given size.
+                "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.)
+                "minimal": Scale as least as possible.  (Output size might be smaller than given size.)
+                Defaults to "lower_bound".
+        """
+        print("Params passed to Resize transform:")
+        print("\twidth: ", width)
+        print("\theight: ", height)
+        print("\tresize_target: ", resize_target)
+        print("\tkeep_aspect_ratio: ", keep_aspect_ratio)
+        print("\tensure_multiple_of: ", ensure_multiple_of)
+        print("\tresize_method: ", resize_method)
+
+        self.__width = width
+        self.__height = height
+
+        self.__keep_aspect_ratio = keep_aspect_ratio
+        self.__multiple_of = ensure_multiple_of
+        self.__resize_method = resize_method
+
+    def constrain_to_multiple_of(self, x, min_val=0, max_val=None):
+        y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int)
+
+        if max_val is not None and y > max_val:
+            y = (np.floor(x / self.__multiple_of)
+                 * self.__multiple_of).astype(int)
+
+        if y < min_val:
+            y = (np.ceil(x / self.__multiple_of)
+                 * self.__multiple_of).astype(int)
+
+        return y
+
+    def get_size(self, width, height):
+        # determine new height and width
+        scale_height = self.__height / height
+        scale_width = self.__width / width
+
+        if self.__keep_aspect_ratio:
+            if self.__resize_method == "lower_bound":
+                # scale such that output size is lower bound
+                if scale_width > scale_height:
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            elif self.__resize_method == "upper_bound":
+                # scale such that output size is upper bound
+                if scale_width < scale_height:
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            elif self.__resize_method == "minimal":
+                # scale as least as possbile
+                if abs(1 - scale_width) < abs(1 - scale_height):
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            else:
+                raise ValueError(
+                    f"resize_method {self.__resize_method} not implemented"
+                )
+
+        if self.__resize_method == "lower_bound":
+            new_height = self.constrain_to_multiple_of(
+                scale_height * height, min_val=self.__height
+            )
+            new_width = self.constrain_to_multiple_of(
+                scale_width * width, min_val=self.__width
+            )
+        elif self.__resize_method == "upper_bound":
+            new_height = self.constrain_to_multiple_of(
+                scale_height * height, max_val=self.__height
+            )
+            new_width = self.constrain_to_multiple_of(
+                scale_width * width, max_val=self.__width
+            )
+        elif self.__resize_method == "minimal":
+            new_height = self.constrain_to_multiple_of(scale_height * height)
+            new_width = self.constrain_to_multiple_of(scale_width * width)
+        else:
+            raise ValueError(
+                f"resize_method {self.__resize_method} not implemented")
+
+        return (new_width, new_height)
+
+    def __call__(self, x):
+        width, height = self.get_size(*x.shape[-2:][::-1])
+        return nn.functional.interpolate(x, (height, width), mode='bilinear', align_corners=True)
+
+class PrepForMidas(object):
+    def __init__(self, resize_mode="minimal", keep_aspect_ratio=True, img_size=384, do_resize=True):
+        if isinstance(img_size, int):
+            img_size = (img_size, img_size)
+        net_h, net_w = img_size
+        self.normalization = Normalize(
+            mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
+        self.resizer = Resize(net_w, net_h, keep_aspect_ratio=keep_aspect_ratio, ensure_multiple_of=32, resize_method=resize_mode) \
+            if do_resize else nn.Identity()
+
+    def __call__(self, x):
+        return self.normalization(self.resizer(x))
+
+
+class MidasCore(nn.Module):
+    def __init__(self, midas, trainable=False, fetch_features=True, layer_names=('out_conv', 'l4_rn', 'r4', 'r3', 'r2', 'r1'), freeze_bn=False, keep_aspect_ratio=True,
+                 img_size=384, **kwargs):
+        """Midas Base model used for multi-scale feature extraction.
+
+        Args:
+            midas (torch.nn.Module): Midas model.
+            trainable (bool, optional): Train midas model. Defaults to False.
+            fetch_features (bool, optional): Extract multi-scale features. Defaults to True.
+            layer_names (tuple, optional): Layers used for feature extraction. Order = (head output features, last layer features, ...decoder features). Defaults to ('out_conv', 'l4_rn', 'r4', 'r3', 'r2', 'r1').
+            freeze_bn (bool, optional): Freeze BatchNorm. Generally results in better finetuning performance. Defaults to False.
+            keep_aspect_ratio (bool, optional): Keep the aspect ratio of input images while resizing. Defaults to True.
+            img_size (int, tuple, optional): Input resolution. Defaults to 384.
+        """
+        super().__init__()
+        self.core = midas
+        self.output_channels = None
+        self.core_out = {}
+        self.trainable = trainable
+        self.fetch_features = fetch_features
+        # midas.scratch.output_conv = nn.Identity()
+        self.handles = []
+        # self.layer_names = ['out_conv','l4_rn', 'r4', 'r3', 'r2', 'r1']
+        self.layer_names = layer_names
+
+        self.set_trainable(trainable)
+        self.set_fetch_features(fetch_features)
+
+        self.prep = PrepForMidas(keep_aspect_ratio=keep_aspect_ratio,
+                                 img_size=img_size, do_resize=kwargs.get('do_resize', True))
+
+        if freeze_bn:
+            self.freeze_bn()
+
+    def set_trainable(self, trainable):
+        self.trainable = trainable
+        if trainable:
+            self.unfreeze()
+        else:
+            self.freeze()
+        return self
+
+    def set_fetch_features(self, fetch_features):
+        self.fetch_features = fetch_features
+        if fetch_features:
+            if len(self.handles) == 0:
+                self.attach_hooks(self.core)
+        else:
+            self.remove_hooks()
+        return self
+
+    def freeze(self):
+        for p in self.parameters():
+            p.requires_grad = False
+        self.trainable = False
+        return self
+
+    def unfreeze(self):
+        for p in self.parameters():
+            p.requires_grad = True
+        self.trainable = True
+        return self
+
+    def freeze_bn(self):
+        for m in self.modules():
+            if isinstance(m, nn.BatchNorm2d):
+                m.eval()
+        return self
+
+    def forward(self, x, denorm=False, return_rel_depth=False):
+        with torch.no_grad():
+            if denorm:
+                x = denormalize(x)
+            x = self.prep(x)
+            # print("Shape after prep: ", x.shape)
+
+        with torch.set_grad_enabled(self.trainable):
+
+            # print("Input size to Midascore", x.shape)
+            rel_depth = self.core(x)
+            # print("Output from midas shape", rel_depth.shape)
+            if not self.fetch_features:
+                return rel_depth
+        out = [self.core_out[k] for k in self.layer_names]
+
+        if return_rel_depth:
+            return rel_depth, out
+        return out
+
+    def get_rel_pos_params(self):
+        for name, p in self.core.pretrained.named_parameters():
+            if "relative_position" in name:
+                yield p
+
+    def get_enc_params_except_rel_pos(self):
+        for name, p in self.core.pretrained.named_parameters():
+            if "relative_position" not in name:
+                yield p
+
+    def freeze_encoder(self, freeze_rel_pos=False):
+        if freeze_rel_pos:
+            for p in self.core.pretrained.parameters():
+                p.requires_grad = False
+        else:
+            for p in self.get_enc_params_except_rel_pos():
+                p.requires_grad = False
+        return self
+
+    def attach_hooks(self, midas):
+        if len(self.handles) > 0:
+            self.remove_hooks()
+        if "out_conv" in self.layer_names:
+            self.handles.append(list(midas.scratch.output_conv.children())[
+                                3].register_forward_hook(get_activation("out_conv", self.core_out)))
+        if "r4" in self.layer_names:
+            self.handles.append(midas.scratch.refinenet4.register_forward_hook(
+                get_activation("r4", self.core_out)))
+        if "r3" in self.layer_names:
+            self.handles.append(midas.scratch.refinenet3.register_forward_hook(
+                get_activation("r3", self.core_out)))
+        if "r2" in self.layer_names:
+            self.handles.append(midas.scratch.refinenet2.register_forward_hook(
+                get_activation("r2", self.core_out)))
+        if "r1" in self.layer_names:
+            self.handles.append(midas.scratch.refinenet1.register_forward_hook(
+                get_activation("r1", self.core_out)))
+        if "l4_rn" in self.layer_names:
+            self.handles.append(midas.scratch.layer4_rn.register_forward_hook(
+                get_activation("l4_rn", self.core_out)))
+
+        return self
+
+    def remove_hooks(self):
+        for h in self.handles:
+            h.remove()
+        return self
+
+    def __del__(self):
+        self.remove_hooks()
+
+    def set_output_channels(self, model_type):
+        self.output_channels = MIDAS_SETTINGS[model_type]
+
+    @staticmethod
+    def build(midas_model_type="DPT_BEiT_L_384", train_midas=False, use_pretrained_midas=True, fetch_features=False, freeze_bn=True, force_keep_ar=False, force_reload=False, **kwargs):
+        if midas_model_type not in MIDAS_SETTINGS:
+            raise ValueError(
+                f"Invalid model type: {midas_model_type}. Must be one of {list(MIDAS_SETTINGS.keys())}")
+        if "img_size" in kwargs:
+            kwargs = MidasCore.parse_img_size(kwargs)
+        img_size = kwargs.pop("img_size", [384, 384])
+        print("img_size", img_size)
+        midas = torch.hub.load("intel-isl/MiDaS", midas_model_type,
+                               pretrained=use_pretrained_midas, force_reload=force_reload)
+        kwargs.update({'keep_aspect_ratio': force_keep_ar})
+        midas_core = MidasCore(midas, trainable=train_midas, fetch_features=fetch_features,
+                               freeze_bn=freeze_bn, img_size=img_size, **kwargs)
+        midas_core.set_output_channels(midas_model_type)
+        return midas_core
+
+    @staticmethod
+    def build_from_config(config):
+        return MidasCore.build(**config)
+
+    @staticmethod
+    def parse_img_size(config):
+        assert 'img_size' in config
+        if isinstance(config['img_size'], str):
+            assert "," in config['img_size'], "img_size should be a string with comma separated img_size=H,W"
+            config['img_size'] = list(map(int, config['img_size'].split(",")))
+            assert len(
+                config['img_size']) == 2, "img_size should be a string with comma separated img_size=H,W"
+        elif isinstance(config['img_size'], int):
+            config['img_size'] = [config['img_size'], config['img_size']]
+        else:
+            assert isinstance(config['img_size'], list) and len(
+                config['img_size']) == 2, "img_size should be a list of H,W"
+        return config
+
+
+nchannels2models = {
+    tuple([256]*5): ["DPT_BEiT_L_384", "DPT_BEiT_L_512", "DPT_BEiT_B_384", "DPT_SwinV2_L_384", "DPT_SwinV2_B_384", "DPT_SwinV2_T_256", "DPT_Large", "DPT_Hybrid"],
+    (512, 256, 128, 64, 64): ["MiDaS_small"]
+}
+
+# Model name to number of output channels
+MIDAS_SETTINGS = {m: k for k, v in nchannels2models.items()
+                  for m in v
+                  }

ZoeDepth/zoedepth/models/builder.py

@@ -0,0 +1,51 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+from importlib import import_module
+from zoedepth.models.depth_model import DepthModel
+
+def build_model(config) -> DepthModel:
+    """Builds a model from a config. The model is specified by the model name and version in the config. The model is then constructed using the build_from_config function of the model interface.
+    This function should be used to construct models for training and evaluation.
+
+    Args:
+        config (dict): Config dict. Config is constructed in utils/config.py. Each model has its own config file(s) saved in its root model folder.
+
+    Returns:
+        torch.nn.Module: Model corresponding to name and version as specified in config
+    """
+    module_name = f"zoedepth.models.{config.model}"
+    try:
+        module = import_module(module_name)
+    except ModuleNotFoundError as e:
+        # print the original error message
+        print(e)
+        raise ValueError(
+            f"Model {config.model} not found. Refer above error for details.") from e
+    try:
+        get_version = getattr(module, "get_version")
+    except AttributeError as e:
+        raise ValueError(
+            f"Model {config.model} has no get_version function.") from e
+    return get_version(config.version_name).build_from_config(config)

ZoeDepth/zoedepth/models/depth_model.py

@@ -0,0 +1,152 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torchvision import transforms
+import PIL.Image
+from PIL import Image
+from typing import Union
+
+
+class DepthModel(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.device = 'cpu'
+    
+    def to(self, device) -> nn.Module:
+        self.device = device
+        return super().to(device)
+    
+    def forward(self, x, *args, **kwargs):
+        raise NotImplementedError
+    
+    def _infer(self, x: torch.Tensor):
+        """
+        Inference interface for the model
+        Args:
+            x (torch.Tensor): input tensor of shape (b, c, h, w)
+        Returns:
+            torch.Tensor: output tensor of shape (b, 1, h, w)
+        """
+        return self(x)['metric_depth']
+    
+    def _infer_with_pad_aug(self, x: torch.Tensor, pad_input: bool=True, fh: float=3, fw: float=3, upsampling_mode: str='bicubic', padding_mode="reflect", **kwargs) -> torch.Tensor:
+        """
+        Inference interface for the model with padding augmentation
+        Padding augmentation fixes the boundary artifacts in the output depth map.
+        Boundary artifacts are sometimes caused by the fact that the model is trained on NYU raw dataset which has a black or white border around the image.
+        This augmentation pads the input image and crops the prediction back to the original size / view.
+
+        Note: This augmentation is not required for the models trained with 'avoid_boundary'=True.
+        Args:
+            x (torch.Tensor): input tensor of shape (b, c, h, w)
+            pad_input (bool, optional): whether to pad the input or not. Defaults to True.
+            fh (float, optional): height padding factor. The padding is calculated as sqrt(h/2) * fh. Defaults to 3.
+            fw (float, optional): width padding factor. The padding is calculated as sqrt(w/2) * fw. Defaults to 3.
+            upsampling_mode (str, optional): upsampling mode. Defaults to 'bicubic'.
+            padding_mode (str, optional): padding mode. Defaults to "reflect".
+        Returns:
+            torch.Tensor: output tensor of shape (b, 1, h, w)
+        """
+        # assert x is nchw and c = 3
+        assert x.dim() == 4, "x must be 4 dimensional, got {}".format(x.dim())
+        assert x.shape[1] == 3, "x must have 3 channels, got {}".format(x.shape[1])
+
+        if pad_input:
+            assert fh > 0 or fw > 0, "atlease one of fh and fw must be greater than 0"
+            pad_h = int(np.sqrt(x.shape[2]/2) * fh)
+            pad_w = int(np.sqrt(x.shape[3]/2) * fw)
+            padding = [pad_w, pad_w]
+            if pad_h > 0:
+                padding += [pad_h, pad_h]
+            
+            x = F.pad(x, padding, mode=padding_mode, **kwargs)
+        out = self._infer(x)
+        if out.shape[-2:] != x.shape[-2:]:
+            out = F.interpolate(out, size=(x.shape[2], x.shape[3]), mode=upsampling_mode, align_corners=False)
+        if pad_input:
+            # crop to the original size, handling the case where pad_h and pad_w is 0
+            if pad_h > 0:
+                out = out[:, :, pad_h:-pad_h,:]
+            if pad_w > 0:
+                out = out[:, :, :, pad_w:-pad_w]
+        return out
+    
+    def infer_with_flip_aug(self, x, pad_input: bool=True, **kwargs) -> torch.Tensor:
+        """
+        Inference interface for the model with horizontal flip augmentation
+        Horizontal flip augmentation improves the accuracy of the model by averaging the output of the model with and without horizontal flip.
+        Args:
+            x (torch.Tensor): input tensor of shape (b, c, h, w)
+            pad_input (bool, optional): whether to use padding augmentation. Defaults to True.
+        Returns:
+            torch.Tensor: output tensor of shape (b, 1, h, w)
+        """
+        # infer with horizontal flip and average
+        out = self._infer_with_pad_aug(x, pad_input=pad_input, **kwargs)
+        out_flip = self._infer_with_pad_aug(torch.flip(x, dims=[3]), pad_input=pad_input, **kwargs)
+        out = (out + torch.flip(out_flip, dims=[3])) / 2
+        return out
+    
+    def infer(self, x, pad_input: bool=True, with_flip_aug: bool=True, **kwargs) -> torch.Tensor:
+        """
+        Inference interface for the model
+        Args:
+            x (torch.Tensor): input tensor of shape (b, c, h, w)
+            pad_input (bool, optional): whether to use padding augmentation. Defaults to True.
+            with_flip_aug (bool, optional): whether to use horizontal flip augmentation. Defaults to True.
+        Returns:
+            torch.Tensor: output tensor of shape (b, 1, h, w)
+        """
+        if with_flip_aug:
+            return self.infer_with_flip_aug(x, pad_input=pad_input, **kwargs)
+        else:
+            return self._infer_with_pad_aug(x, pad_input=pad_input, **kwargs)
+    
+    @torch.no_grad()
+    def infer_pil(self, pil_img, pad_input: bool=True, with_flip_aug: bool=True, output_type: str="numpy", **kwargs) -> Union[np.ndarray, PIL.Image.Image, torch.Tensor]:
+        """
+        Inference interface for the model for PIL image
+        Args:
+            pil_img (PIL.Image.Image): input PIL image
+            pad_input (bool, optional): whether to use padding augmentation. Defaults to True.
+            with_flip_aug (bool, optional): whether to use horizontal flip augmentation. Defaults to True.
+            output_type (str, optional): output type. Supported values are 'numpy', 'pil' and 'tensor'. Defaults to "numpy".
+        """
+        x = transforms.ToTensor()(pil_img).unsqueeze(0).to(self.device)
+        out_tensor = self.infer(x, pad_input=pad_input, with_flip_aug=with_flip_aug, **kwargs)
+        if output_type == "numpy":
+            return out_tensor.squeeze().cpu().numpy()
+        elif output_type == "pil":
+            # uint16 is required for depth pil image
+            out_16bit_numpy = (out_tensor.squeeze().cpu().numpy()*256).astype(np.uint16)
+            return Image.fromarray(out_16bit_numpy)
+        elif output_type == "tensor":
+            return out_tensor.squeeze().cpu()
+        else:
+            raise ValueError(f"output_type {output_type} not supported. Supported values are 'numpy', 'pil' and 'tensor'")
+    

ZoeDepth/zoedepth/models/layers/attractor.py

@@ -0,0 +1,208 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import torch
+import torch.nn as nn
+
+
+@torch.jit.script
+def exp_attractor(dx, alpha: float = 300, gamma: int = 2):
+    """Exponential attractor: dc = exp(-alpha*|dx|^gamma) * dx , where dx = a - c, a = attractor point, c = bin center, dc = shift in bin centermmary for exp_attractor
+
+    Args:
+        dx (torch.Tensor): The difference tensor dx = Ai - Cj, where Ai is the attractor point and Cj is the bin center.
+        alpha (float, optional): Proportional Attractor strength. Determines the absolute strength. Lower alpha = greater attraction. Defaults to 300.
+        gamma (int, optional): Exponential Attractor strength. Determines the "region of influence" and indirectly number of bin centers affected. Lower gamma = farther reach. Defaults to 2.
+
+    Returns:
+        torch.Tensor : Delta shifts - dc; New bin centers = Old bin centers + dc
+    """
+    return torch.exp(-alpha*(torch.abs(dx)**gamma)) * (dx)
+
+
+@torch.jit.script
+def inv_attractor(dx, alpha: float = 300, gamma: int = 2):
+    """Inverse attractor: dc = dx / (1 + alpha*dx^gamma), where dx = a - c, a = attractor point, c = bin center, dc = shift in bin center
+    This is the default one according to the accompanying paper. 
+
+    Args:
+        dx (torch.Tensor): The difference tensor dx = Ai - Cj, where Ai is the attractor point and Cj is the bin center.
+        alpha (float, optional): Proportional Attractor strength. Determines the absolute strength. Lower alpha = greater attraction. Defaults to 300.
+        gamma (int, optional): Exponential Attractor strength. Determines the "region of influence" and indirectly number of bin centers affected. Lower gamma = farther reach. Defaults to 2.
+
+    Returns:
+        torch.Tensor: Delta shifts - dc; New bin centers = Old bin centers + dc
+    """
+    return dx.div(1+alpha*dx.pow(gamma))
+
+
+class AttractorLayer(nn.Module):
+    def __init__(self, in_features, n_bins, n_attractors=16, mlp_dim=128, min_depth=1e-3, max_depth=10,
+                 alpha=300, gamma=2, kind='sum', attractor_type='exp', memory_efficient=False):
+        """
+        Attractor layer for bin centers. Bin centers are bounded on the interval (min_depth, max_depth)
+        """
+        super().__init__()
+
+        self.n_attractors = n_attractors
+        self.n_bins = n_bins
+        self.min_depth = min_depth
+        self.max_depth = max_depth
+        self.alpha = alpha
+        self.gamma = gamma
+        self.kind = kind
+        self.attractor_type = attractor_type
+        self.memory_efficient = memory_efficient
+
+        self._net = nn.Sequential(
+            nn.Conv2d(in_features, mlp_dim, 1, 1, 0),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(mlp_dim, n_attractors*2, 1, 1, 0),  # x2 for linear norm
+            nn.ReLU(inplace=True)
+        )
+
+    def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False):
+        """
+        Args:
+            x (torch.Tensor) : feature block; shape - n, c, h, w
+            b_prev (torch.Tensor) : previous bin centers normed; shape - n, prev_nbins, h, w
+        
+        Returns:
+            tuple(torch.Tensor,torch.Tensor) : new bin centers normed and scaled; shape - n, nbins, h, w
+        """
+        if prev_b_embedding is not None:
+            if interpolate:
+                prev_b_embedding = nn.functional.interpolate(
+                    prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True)
+            x = x + prev_b_embedding
+
+        A = self._net(x)
+        eps = 1e-3
+        A = A + eps
+        n, c, h, w = A.shape
+        A = A.view(n, self.n_attractors, 2, h, w)
+        A_normed = A / A.sum(dim=2, keepdim=True)  # n, a, 2, h, w
+        A_normed = A[:, :, 0, ...]  # n, na, h, w
+
+        b_prev = nn.functional.interpolate(
+            b_prev, (h, w), mode='bilinear', align_corners=True)
+        b_centers = b_prev
+
+        if self.attractor_type == 'exp':
+            dist = exp_attractor
+        else:
+            dist = inv_attractor
+
+        if not self.memory_efficient:
+            func = {'mean': torch.mean, 'sum': torch.sum}[self.kind]
+            # .shape N, nbins, h, w
+            delta_c = func(dist(A_normed.unsqueeze(
+                2) - b_centers.unsqueeze(1)), dim=1)
+        else:
+            delta_c = torch.zeros_like(b_centers, device=b_centers.device)
+            for i in range(self.n_attractors):
+                # .shape N, nbins, h, w
+                delta_c += dist(A_normed[:, i, ...].unsqueeze(1) - b_centers)
+
+            if self.kind == 'mean':
+                delta_c = delta_c / self.n_attractors
+
+        b_new_centers = b_centers + delta_c
+        B_centers = (self.max_depth - self.min_depth) * \
+            b_new_centers + self.min_depth
+        B_centers, _ = torch.sort(B_centers, dim=1)
+        B_centers = torch.clip(B_centers, self.min_depth, self.max_depth)
+        return b_new_centers, B_centers
+
+
+class AttractorLayerUnnormed(nn.Module):
+    def __init__(self, in_features, n_bins, n_attractors=16, mlp_dim=128, min_depth=1e-3, max_depth=10,
+                 alpha=300, gamma=2, kind='sum', attractor_type='exp', memory_efficient=False):
+        """
+        Attractor layer for bin centers. Bin centers are unbounded
+        """
+        super().__init__()
+
+        self.n_attractors = n_attractors
+        self.n_bins = n_bins
+        self.min_depth = min_depth
+        self.max_depth = max_depth
+        self.alpha = alpha
+        self.gamma = gamma
+        self.kind = kind
+        self.attractor_type = attractor_type
+        self.memory_efficient = memory_efficient
+
+        self._net = nn.Sequential(
+            nn.Conv2d(in_features, mlp_dim, 1, 1, 0),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(mlp_dim, n_attractors, 1, 1, 0),
+            nn.Softplus()
+        )
+
+    def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False):
+        """
+        Args:
+            x (torch.Tensor) : feature block; shape - n, c, h, w
+            b_prev (torch.Tensor) : previous bin centers normed; shape - n, prev_nbins, h, w
+        
+        Returns:
+            tuple(torch.Tensor,torch.Tensor) : new bin centers unbounded; shape - n, nbins, h, w. Two outputs just to keep the API consistent with the normed version
+        """
+        if prev_b_embedding is not None:
+            if interpolate:
+                prev_b_embedding = nn.functional.interpolate(
+                    prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True)
+            x = x + prev_b_embedding
+
+        A = self._net(x)
+        n, c, h, w = A.shape
+
+        b_prev = nn.functional.interpolate(
+            b_prev, (h, w), mode='bilinear', align_corners=True)
+        b_centers = b_prev
+
+        if self.attractor_type == 'exp':
+            dist = exp_attractor
+        else:
+            dist = inv_attractor
+
+        if not self.memory_efficient:
+            func = {'mean': torch.mean, 'sum': torch.sum}[self.kind]
+            # .shape N, nbins, h, w
+            delta_c = func(
+                dist(A.unsqueeze(2) - b_centers.unsqueeze(1)), dim=1)
+        else:
+            delta_c = torch.zeros_like(b_centers, device=b_centers.device)
+            for i in range(self.n_attractors):
+                delta_c += dist(A[:, i, ...].unsqueeze(1) -
+                                b_centers)  # .shape N, nbins, h, w
+
+            if self.kind == 'mean':
+                delta_c = delta_c / self.n_attractors
+
+        b_new_centers = b_centers + delta_c
+        B_centers = b_new_centers
+
+        return b_new_centers, B_centers

ZoeDepth/zoedepth/models/layers/dist_layers.py

@@ -0,0 +1,121 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import torch
+import torch.nn as nn
+
+
+def log_binom(n, k, eps=1e-7):
+    """ log(nCk) using stirling approximation """
+    n = n + eps
+    k = k + eps
+    return n * torch.log(n) - k * torch.log(k) - (n-k) * torch.log(n-k+eps)
+
+
+class LogBinomial(nn.Module):
+    def __init__(self, n_classes=256, act=torch.softmax):
+        """Compute log binomial distribution for n_classes
+
+        Args:
+            n_classes (int, optional): number of output classes. Defaults to 256.
+        """
+        super().__init__()
+        self.K = n_classes
+        self.act = act
+        self.register_buffer('k_idx', torch.arange(
+            0, n_classes).view(1, -1, 1, 1))
+        self.register_buffer('K_minus_1', torch.Tensor(
+            [self.K-1]).view(1, -1, 1, 1))
+
+    def forward(self, x, t=1., eps=1e-4):
+        """Compute log binomial distribution for x
+
+        Args:
+            x (torch.Tensor - NCHW): probabilities
+            t (float, torch.Tensor - NCHW, optional): Temperature of distribution. Defaults to 1..
+            eps (float, optional): Small number for numerical stability. Defaults to 1e-4.
+
+        Returns:
+            torch.Tensor -NCHW: log binomial distribution logbinomial(p;t)
+        """
+        if x.ndim == 3:
+            x = x.unsqueeze(1)  # make it nchw
+
+        one_minus_x = torch.clamp(1 - x, eps, 1)
+        x = torch.clamp(x, eps, 1)
+        y = log_binom(self.K_minus_1, self.k_idx) + self.k_idx * \
+            torch.log(x) + (self.K - 1 - self.k_idx) * torch.log(one_minus_x)
+        return self.act(y/t, dim=1)
+
+
+class ConditionalLogBinomial(nn.Module):
+    def __init__(self, in_features, condition_dim, n_classes=256, bottleneck_factor=2, p_eps=1e-4, max_temp=50, min_temp=1e-7, act=torch.softmax):
+        """Conditional Log Binomial distribution
+
+        Args:
+            in_features (int): number of input channels in main feature
+            condition_dim (int): number of input channels in condition feature
+            n_classes (int, optional): Number of classes. Defaults to 256.
+            bottleneck_factor (int, optional): Hidden dim factor. Defaults to 2.
+            p_eps (float, optional): small eps value. Defaults to 1e-4.
+            max_temp (float, optional): Maximum temperature of output distribution. Defaults to 50.
+            min_temp (float, optional): Minimum temperature of output distribution. Defaults to 1e-7.
+        """
+        super().__init__()
+        self.p_eps = p_eps
+        self.max_temp = max_temp
+        self.min_temp = min_temp
+        self.log_binomial_transform = LogBinomial(n_classes, act=act)
+        bottleneck = (in_features + condition_dim) // bottleneck_factor
+        self.mlp = nn.Sequential(
+            nn.Conv2d(in_features + condition_dim, bottleneck,
+                      kernel_size=1, stride=1, padding=0),
+            nn.GELU(),
+            # 2 for p linear norm, 2 for t linear norm
+            nn.Conv2d(bottleneck, 2+2, kernel_size=1, stride=1, padding=0),
+            nn.Softplus()
+        )
+
+    def forward(self, x, cond):
+        """Forward pass
+
+        Args:
+            x (torch.Tensor - NCHW): Main feature
+            cond (torch.Tensor - NCHW): condition feature
+
+        Returns:
+            torch.Tensor: Output log binomial distribution
+        """
+        pt = self.mlp(torch.concat((x, cond), dim=1))
+        p, t = pt[:, :2, ...], pt[:, 2:, ...]
+
+        p = p + self.p_eps
+        p = p[:, 0, ...] / (p[:, 0, ...] + p[:, 1, ...])
+
+        t = t + self.p_eps
+        t = t[:, 0, ...] / (t[:, 0, ...] + t[:, 1, ...])
+        t = t.unsqueeze(1)
+        t = (self.max_temp - self.min_temp) * t + self.min_temp
+
+        return self.log_binomial_transform(p, t)

ZoeDepth/zoedepth/models/layers/localbins_layers.py

@@ -0,0 +1,169 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import torch
+import torch.nn as nn
+
+
+class SeedBinRegressor(nn.Module):
+    def __init__(self, in_features, n_bins=16, mlp_dim=256, min_depth=1e-3, max_depth=10):
+        """Bin center regressor network. Bin centers are bounded on (min_depth, max_depth) interval.
+
+        Args:
+            in_features (int): input channels
+            n_bins (int, optional): Number of bin centers. Defaults to 16.
+            mlp_dim (int, optional): Hidden dimension. Defaults to 256.
+            min_depth (float, optional): Min depth value. Defaults to 1e-3.
+            max_depth (float, optional): Max depth value. Defaults to 10.
+        """
+        super().__init__()
+        self.version = "1_1"
+        self.min_depth = min_depth
+        self.max_depth = max_depth
+
+        self._net = nn.Sequential(
+            nn.Conv2d(in_features, mlp_dim, 1, 1, 0),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(mlp_dim, n_bins, 1, 1, 0),
+            nn.ReLU(inplace=True)
+        )
+
+    def forward(self, x):
+        """
+        Returns tensor of bin_width vectors (centers). One vector b for every pixel
+        """
+        B = self._net(x)
+        eps = 1e-3
+        B = B + eps
+        B_widths_normed = B / B.sum(dim=1, keepdim=True)
+        B_widths = (self.max_depth - self.min_depth) * \
+            B_widths_normed  # .shape NCHW
+        # pad has the form (left, right, top, bottom, front, back)
+        B_widths = nn.functional.pad(
+            B_widths, (0, 0, 0, 0, 1, 0), mode='constant', value=self.min_depth)
+        B_edges = torch.cumsum(B_widths, dim=1)  # .shape NCHW
+
+        B_centers = 0.5 * (B_edges[:, :-1, ...] + B_edges[:, 1:, ...])
+        return B_widths_normed, B_centers
+
+
+class SeedBinRegressorUnnormed(nn.Module):
+    def __init__(self, in_features, n_bins=16, mlp_dim=256, min_depth=1e-3, max_depth=10):
+        """Bin center regressor network. Bin centers are unbounded
+
+        Args:
+            in_features (int): input channels
+            n_bins (int, optional): Number of bin centers. Defaults to 16.
+            mlp_dim (int, optional): Hidden dimension. Defaults to 256.
+            min_depth (float, optional): Not used. (for compatibility with SeedBinRegressor)
+            max_depth (float, optional): Not used. (for compatibility with SeedBinRegressor)
+        """
+        super().__init__()
+        self.version = "1_1"
+        self._net = nn.Sequential(
+            nn.Conv2d(in_features, mlp_dim, 1, 1, 0),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(mlp_dim, n_bins, 1, 1, 0),
+            nn.Softplus()
+        )
+
+    def forward(self, x):
+        """
+        Returns tensor of bin_width vectors (centers). One vector b for every pixel
+        """
+        B_centers = self._net(x)
+        return B_centers, B_centers
+
+
+class Projector(nn.Module):
+    def __init__(self, in_features, out_features, mlp_dim=128):
+        """Projector MLP
+
+        Args:
+            in_features (int): input channels
+            out_features (int): output channels
+            mlp_dim (int, optional): hidden dimension. Defaults to 128.
+        """
+        super().__init__()
+
+        self._net = nn.Sequential(
+            nn.Conv2d(in_features, mlp_dim, 1, 1, 0),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(mlp_dim, out_features, 1, 1, 0),
+        )
+
+    def forward(self, x):
+        return self._net(x)
+
+
+
+class LinearSplitter(nn.Module):
+    def __init__(self, in_features, prev_nbins, split_factor=2, mlp_dim=128, min_depth=1e-3, max_depth=10):
+        super().__init__()
+
+        self.prev_nbins = prev_nbins
+        self.split_factor = split_factor
+        self.min_depth = min_depth
+        self.max_depth = max_depth
+
+        self._net = nn.Sequential(
+            nn.Conv2d(in_features, mlp_dim, 1, 1, 0),
+            nn.GELU(),
+            nn.Conv2d(mlp_dim, prev_nbins * split_factor, 1, 1, 0),
+            nn.ReLU()
+        )
+    
+    def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False):
+        """
+        x : feature block; shape - n, c, h, w
+        b_prev : previous bin widths normed; shape - n, prev_nbins, h, w
+        """
+        if prev_b_embedding is not None:
+            if interpolate:
+                prev_b_embedding = nn.functional.interpolate(prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True)
+            x = x + prev_b_embedding
+        S = self._net(x)
+        eps = 1e-3
+        S = S + eps
+        n, c, h, w = S.shape
+        S = S.view(n, self.prev_nbins, self.split_factor, h, w)
+        S_normed = S / S.sum(dim=2, keepdim=True)  # fractional splits
+
+        b_prev = nn.functional.interpolate(b_prev, (h,w), mode='bilinear', align_corners=True)
+        
+
+        b_prev = b_prev / b_prev.sum(dim=1, keepdim=True)  # renormalize for gurantees
+        # print(b_prev.shape, S_normed.shape)
+        # if is_for_query:(1).expand(-1, b_prev.size(0)//n, -1, -1, -1, -1).flatten(0,1)  # TODO ? can replace all this with a single torch.repeat?
+        b = b_prev.unsqueeze(2) * S_normed
+        b = b.flatten(1,2)  # .shape n, prev_nbins * split_factor, h, w
+
+        # calculate bin centers for loss calculation
+        B_widths = (self.max_depth - self.min_depth) * b  # .shape N, nprev * splitfactor, H, W
+        # pad has the form (left, right, top, bottom, front, back)
+        B_widths = nn.functional.pad(B_widths, (0,0,0,0,1,0), mode='constant', value=self.min_depth)
+        B_edges = torch.cumsum(B_widths, dim=1)  # .shape NCHW
+
+        B_centers = 0.5 * (B_edges[:, :-1, ...] + B_edges[:,1:,...])
+        return b, B_centers

ZoeDepth/zoedepth/models/layers/patch_transformer.py

@@ -0,0 +1,91 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import torch
+import torch.nn as nn
+
+
+class PatchTransformerEncoder(nn.Module):
+    def __init__(self, in_channels, patch_size=10, embedding_dim=128, num_heads=4, use_class_token=False):
+        """ViT-like transformer block
+
+        Args:
+            in_channels (int): Input channels
+            patch_size (int, optional): patch size. Defaults to 10.
+            embedding_dim (int, optional): Embedding dimension in transformer model. Defaults to 128.
+            num_heads (int, optional): number of attention heads. Defaults to 4.
+            use_class_token (bool, optional): Whether to use extra token at the start for global accumulation (called as "class token"). Defaults to False.
+        """
+        super(PatchTransformerEncoder, self).__init__()
+        self.use_class_token = use_class_token
+        encoder_layers = nn.TransformerEncoderLayer(
+            embedding_dim, num_heads, dim_feedforward=1024)
+        self.transformer_encoder = nn.TransformerEncoder(
+            encoder_layers, num_layers=4)  # takes shape S,N,E
+
+        self.embedding_convPxP = nn.Conv2d(in_channels, embedding_dim,
+                                           kernel_size=patch_size, stride=patch_size, padding=0)
+        
+    def positional_encoding_1d(self, sequence_length, batch_size, embedding_dim, device='cpu'):
+        """Generate positional encodings
+
+        Args:
+            sequence_length (int): Sequence length
+            embedding_dim (int): Embedding dimension
+
+        Returns:
+            torch.Tensor SBE: Positional encodings
+        """
+        position = torch.arange(
+            0, sequence_length, dtype=torch.float32, device=device).unsqueeze(1)
+        index = torch.arange(
+            0, embedding_dim, 2, dtype=torch.float32, device=device).unsqueeze(0)
+        div_term = torch.exp(index * (-torch.log(torch.tensor(10000.0, device=device)) / embedding_dim))
+        pos_encoding = position * div_term
+        pos_encoding = torch.cat([torch.sin(pos_encoding), torch.cos(pos_encoding)], dim=1)
+        pos_encoding = pos_encoding.unsqueeze(1).repeat(1, batch_size, 1)
+        return pos_encoding
+        
+
+    def forward(self, x):
+        """Forward pass
+
+        Args:
+            x (torch.Tensor - NCHW): Input feature tensor
+
+        Returns:
+            torch.Tensor - SNE: Transformer output embeddings. S - sequence length (=HW/patch_size^2), N - batch size, E - embedding dim
+        """
+        embeddings = self.embedding_convPxP(x).flatten(
+            2)  # .shape = n,c,s = n, embedding_dim, s
+        if self.use_class_token:
+            # extra special token at start ?
+            embeddings = nn.functional.pad(embeddings, (1, 0))
+        
+        # change to S,N,E format required by transformer
+        embeddings = embeddings.permute(2, 0, 1)
+        S, N, E = embeddings.shape
+        embeddings = embeddings + self.positional_encoding_1d(S, N, E, device=embeddings.device)
+        x = self.transformer_encoder(embeddings)  # .shape = S, N, E
+        return x

ZoeDepth/zoedepth/models/model_io.py

@@ -0,0 +1,92 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import torch
+
+def load_state_dict(model, state_dict):
+    """Load state_dict into model, handling DataParallel and DistributedDataParallel. Also checks for "model" key in state_dict.
+
+    DataParallel prefixes state_dict keys with 'module.' when saving.
+    If the model is not a DataParallel model but the state_dict is, then prefixes are removed.
+    If the model is a DataParallel model but the state_dict is not, then prefixes are added.
+    """
+    state_dict = state_dict.get('model', state_dict)
+    # if model is a DataParallel model, then state_dict keys are prefixed with 'module.'
+
+    do_prefix = isinstance(
+        model, (torch.nn.DataParallel, torch.nn.parallel.DistributedDataParallel))
+    state = {}
+    for k, v in state_dict.items():
+        if k.startswith('module.') and not do_prefix:
+            k = k[7:]
+
+        if not k.startswith('module.') and do_prefix:
+            k = 'module.' + k
+
+        state[k] = v
+
+    model.load_state_dict(state)
+    print("Loaded successfully")
+    return model
+
+
+def load_wts(model, checkpoint_path):
+    ckpt = torch.load(checkpoint_path, map_location='cpu')
+    return load_state_dict(model, ckpt)
+
+
+def load_state_dict_from_url(model, url, **kwargs):
+    state_dict = torch.hub.load_state_dict_from_url(url, map_location='cpu', **kwargs)
+    return load_state_dict(model, state_dict)
+
+
+def load_state_from_resource(model, resource: str):
+    """Loads weights to the model from a given resource. A resource can be of following types:
+        1. URL. Prefixed with "url::"
+                e.g. url::http(s)://url.resource.com/ckpt.pt
+
+        2. Local path. Prefixed with "local::"
+                e.g. local::/path/to/ckpt.pt
+
+
+    Args:
+        model (torch.nn.Module): Model
+        resource (str): resource string
+
+    Returns:
+        torch.nn.Module: Model with loaded weights
+    """
+    print(f"Using pretrained resource {resource}")
+
+    if resource.startswith('url::'):
+        url = resource.split('url::')[1]
+        return load_state_dict_from_url(model, url, progress=True)
+
+    elif resource.startswith('local::'):
+        path = resource.split('local::')[1]
+        return load_wts(model, path)
+        
+    else:
+        raise ValueError("Invalid resource type, only url:: and local:: are supported")
+    

ZoeDepth/zoedepth/models/zoedepth/__init__.py

@@ -0,0 +1,31 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+from .zoedepth_v1 import ZoeDepth 
+
+all_versions = {
+    "v1": ZoeDepth,
+}
+
+get_version = lambda v : all_versions[v]

ZoeDepth/zoedepth/models/zoedepth/config_zoedepth.json

@@ -0,0 +1,58 @@
+{
+    "model": {
+        "name": "ZoeDepth",
+        "version_name": "v1",
+        "n_bins": 64,
+        "bin_embedding_dim": 128,
+        "bin_centers_type": "softplus",
+        "n_attractors":[16, 8, 4, 1],
+        "attractor_alpha": 1000,
+        "attractor_gamma": 2,
+        "attractor_kind" : "mean",
+        "attractor_type" : "inv",
+        "midas_model_type" : "DPT_BEiT_L_384",
+        "min_temp": 0.0212,
+        "max_temp": 50.0,
+        "output_distribution": "logbinomial",
+        "memory_efficient": true,
+        "inverse_midas": false,
+        "img_size": [384, 512]
+    },
+    
+    "train": {
+        "train_midas": true,
+        "use_pretrained_midas": true,
+        "trainer": "zoedepth",
+        "epochs": 5,
+        "bs": 16,
+        "optim_kwargs": {"lr": 0.000161, "wd": 0.01},
+        "sched_kwargs": {"div_factor": 1, "final_div_factor": 10000, "pct_start": 0.7, "three_phase":false, "cycle_momentum": true},
+        "same_lr": false,
+        "w_si": 1,
+        "w_domain": 0.2,
+        "w_reg": 0,
+        "w_grad": 0,
+        "avoid_boundary": false,
+        "random_crop": false,
+        "input_width": 640,
+        "input_height": 480,
+        "midas_lr_factor": 1,
+        "encoder_lr_factor":10,
+        "pos_enc_lr_factor":10,
+        "freeze_midas_bn": true
+
+    },
+
+    "infer":{
+        "train_midas": false,
+        "use_pretrained_midas": false,
+        "pretrained_resource" : "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_N.pt",
+        "force_keep_ar": true
+    },
+
+    "eval":{
+        "train_midas": false,
+        "use_pretrained_midas": false,
+        "pretrained_resource" : "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_N.pt"
+    }
+}

ZoeDepth/zoedepth/models/zoedepth/config_zoedepth_kitti.json

@@ -0,0 +1,22 @@
+{
+    "model": {
+        "bin_centers_type": "normed",
+        "img_size": [384, 768]
+    },
+    
+    "train": {
+    },
+
+    "infer":{
+        "train_midas": false,
+        "use_pretrained_midas": false,
+        "pretrained_resource" : "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_K.pt",
+        "force_keep_ar": true
+    },
+
+    "eval":{
+        "train_midas": false,
+        "use_pretrained_midas": false,
+        "pretrained_resource" : "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_K.pt"
+    }
+}

ZoeDepth/zoedepth/models/zoedepth/zoedepth_v1.py

@@ -0,0 +1,250 @@
+# MIT License
+
+# Copyright (c) 2022 Intelligent Systems Lab Org
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# File author: Shariq Farooq Bhat
+
+import itertools
+
+import torch
+import torch.nn as nn
+from zoedepth.models.depth_model import DepthModel
+from zoedepth.models.base_models.midas import MidasCore
+from zoedepth.models.layers.attractor import AttractorLayer, AttractorLayerUnnormed
+from zoedepth.models.layers.dist_layers import ConditionalLogBinomial
+from zoedepth.models.layers.localbins_layers import (Projector, SeedBinRegressor,
+                                            SeedBinRegressorUnnormed)
+from zoedepth.models.model_io import load_state_from_resource
+
+
+class ZoeDepth(DepthModel):
+    def __init__(self, core,  n_bins=64, bin_centers_type="softplus", bin_embedding_dim=128, min_depth=1e-3, max_depth=10,
+                 n_attractors=[16, 8, 4, 1], attractor_alpha=300, attractor_gamma=2, attractor_kind='sum', attractor_type='exp', min_temp=5, max_temp=50, train_midas=True,
+                 midas_lr_factor=10, encoder_lr_factor=10, pos_enc_lr_factor=10, inverse_midas=False, **kwargs):
+        """ZoeDepth model. This is the version of ZoeDepth that has a single metric head
+
+        Args:
+            core (models.base_models.midas.MidasCore): The base midas model that is used for extraction of "relative" features
+            n_bins (int, optional): Number of bin centers. Defaults to 64.
+            bin_centers_type (str, optional): "normed" or "softplus". Activation type used for bin centers. For "normed" bin centers, linear normalization trick is applied. This results in bounded bin centers.
+                                               For "softplus", softplus activation is used and thus are unbounded. Defaults to "softplus".
+            bin_embedding_dim (int, optional): bin embedding dimension. Defaults to 128.
+            min_depth (float, optional): Lower bound for normed bin centers. Defaults to 1e-3.
+            max_depth (float, optional): Upper bound for normed bin centers. Defaults to 10.
+            n_attractors (List[int], optional): Number of bin attractors at decoder layers. Defaults to [16, 8, 4, 1].
+            attractor_alpha (int, optional): Proportional attractor strength. Refer to models.layers.attractor for more details. Defaults to 300.
+            attractor_gamma (int, optional): Exponential attractor strength. Refer to models.layers.attractor for more details. Defaults to 2.
+            attractor_kind (str, optional): Attraction aggregation "sum" or "mean". Defaults to 'sum'.
+            attractor_type (str, optional): Type of attractor to use; "inv" (Inverse attractor) or "exp" (Exponential attractor). Defaults to 'exp'.
+            min_temp (int, optional): Lower bound for temperature of output probability distribution. Defaults to 5.
+            max_temp (int, optional): Upper bound for temperature of output probability distribution. Defaults to 50.
+            train_midas (bool, optional): Whether to train "core", the base midas model. Defaults to True.
+            midas_lr_factor (int, optional): Learning rate reduction factor for base midas model except its encoder and positional encodings. Defaults to 10.
+            encoder_lr_factor (int, optional): Learning rate reduction factor for the encoder in midas model. Defaults to 10.
+            pos_enc_lr_factor (int, optional): Learning rate reduction factor for positional encodings in the base midas model. Defaults to 10.
+        """
+        super().__init__()
+
+        self.core = core
+        self.max_depth = max_depth
+        self.min_depth = min_depth
+        self.min_temp = min_temp
+        self.bin_centers_type = bin_centers_type
+
+        self.midas_lr_factor = midas_lr_factor
+        self.encoder_lr_factor = encoder_lr_factor
+        self.pos_enc_lr_factor = pos_enc_lr_factor
+        self.train_midas = train_midas
+        self.inverse_midas = inverse_midas
+
+        if self.encoder_lr_factor <= 0:
+            self.core.freeze_encoder(
+                freeze_rel_pos=self.pos_enc_lr_factor <= 0)
+
+        N_MIDAS_OUT = 32
+        btlnck_features = self.core.output_channels[0]
+        num_out_features = self.core.output_channels[1:]
+
+        self.conv2 = nn.Conv2d(btlnck_features, btlnck_features,
+                               kernel_size=1, stride=1, padding=0)  # btlnck conv
+
+        if bin_centers_type == "normed":
+            SeedBinRegressorLayer = SeedBinRegressor
+            Attractor = AttractorLayer
+        elif bin_centers_type == "softplus":
+            SeedBinRegressorLayer = SeedBinRegressorUnnormed
+            Attractor = AttractorLayerUnnormed
+        elif bin_centers_type == "hybrid1":
+            SeedBinRegressorLayer = SeedBinRegressor
+            Attractor = AttractorLayerUnnormed
+        elif bin_centers_type == "hybrid2":
+            SeedBinRegressorLayer = SeedBinRegressorUnnormed
+            Attractor = AttractorLayer
+        else:
+            raise ValueError(
+                "bin_centers_type should be one of 'normed', 'softplus', 'hybrid1', 'hybrid2'")
+
+        self.seed_bin_regressor = SeedBinRegressorLayer(
+            btlnck_features, n_bins=n_bins, min_depth=min_depth, max_depth=max_depth)
+        self.seed_projector = Projector(btlnck_features, bin_embedding_dim)
+        self.projectors = nn.ModuleList([
+            Projector(num_out, bin_embedding_dim)
+            for num_out in num_out_features
+        ])
+        self.attractors = nn.ModuleList([
+            Attractor(bin_embedding_dim, n_bins, n_attractors=n_attractors[i], min_depth=min_depth, max_depth=max_depth,
+                      alpha=attractor_alpha, gamma=attractor_gamma, kind=attractor_kind, attractor_type=attractor_type)
+            for i in range(len(num_out_features))
+        ])
+
+        last_in = N_MIDAS_OUT + 1  # +1 for relative depth
+
+        # use log binomial instead of softmax
+        self.conditional_log_binomial = ConditionalLogBinomial(
+            last_in, bin_embedding_dim, n_classes=n_bins, min_temp=min_temp, max_temp=max_temp)
+
+    def forward(self, x, return_final_centers=False, denorm=False, return_probs=False, **kwargs):
+        """
+        Args:
+            x (torch.Tensor): Input image tensor of shape (B, C, H, W)
+            return_final_centers (bool, optional): Whether to return the final bin centers. Defaults to False.
+            denorm (bool, optional): Whether to denormalize the input image. This reverses ImageNet normalization as midas normalization is different. Defaults to False.
+            return_probs (bool, optional): Whether to return the output probability distribution. Defaults to False.
+        
+        Returns:
+            dict: Dictionary containing the following keys:
+                - rel_depth (torch.Tensor): Relative depth map of shape (B, H, W)
+                - metric_depth (torch.Tensor): Metric depth map of shape (B, 1, H, W)
+                - bin_centers (torch.Tensor): Bin centers of shape (B, n_bins). Present only if return_final_centers is True
+                - probs (torch.Tensor): Output probability distribution of shape (B, n_bins, H, W). Present only if return_probs is True
+
+        """
+        b, c, h, w = x.shape
+        # print("input shape ", x.shape)
+        self.orig_input_width = w
+        self.orig_input_height = h
+        rel_depth, out = self.core(x, denorm=denorm, return_rel_depth=True)
+        # print("output shapes", rel_depth.shape, out.shape)
+
+        outconv_activation = out[0]
+        btlnck = out[1]
+        x_blocks = out[2:]
+
+        x_d0 = self.conv2(btlnck)
+        x = x_d0
+        _, seed_b_centers = self.seed_bin_regressor(x)
+
+        if self.bin_centers_type == 'normed' or self.bin_centers_type == 'hybrid2':
+            b_prev = (seed_b_centers - self.min_depth) / \
+                (self.max_depth - self.min_depth)
+        else:
+            b_prev = seed_b_centers
+
+        prev_b_embedding = self.seed_projector(x)
+
+        # unroll this loop for better performance
+        for projector, attractor, x in zip(self.projectors, self.attractors, x_blocks):
+            b_embedding = projector(x)
+            b, b_centers = attractor(
+                b_embedding, b_prev, prev_b_embedding, interpolate=True)
+            b_prev = b.clone()
+            prev_b_embedding = b_embedding.clone()
+
+        last = outconv_activation
+
+        if self.inverse_midas:
+            # invert depth followed by normalization
+            rel_depth = 1.0 / (rel_depth + 1e-6)
+            rel_depth = (rel_depth - rel_depth.min()) / \
+                (rel_depth.max() - rel_depth.min())
+        # concat rel depth with last. First interpolate rel depth to last size
+        rel_cond = rel_depth.unsqueeze(1)
+        rel_cond = nn.functional.interpolate(
+            rel_cond, size=last.shape[2:], mode='bilinear', align_corners=True)
+        last = torch.cat([last, rel_cond], dim=1)
+
+        b_embedding = nn.functional.interpolate(
+            b_embedding, last.shape[-2:], mode='bilinear', align_corners=True)
+        x = self.conditional_log_binomial(last, b_embedding)
+
+        # Now depth value is Sum px * cx , where cx are bin_centers from the last bin tensor
+        # print(x.shape, b_centers.shape)
+        b_centers = nn.functional.interpolate(
+            b_centers, x.shape[-2:], mode='bilinear', align_corners=True)
+        out = torch.sum(x * b_centers, dim=1, keepdim=True)
+
+        # Structure output dict
+        output = dict(metric_depth=out)
+        if return_final_centers or return_probs:
+            output['bin_centers'] = b_centers
+
+        if return_probs:
+            output['probs'] = x
+
+        return output
+
+    def get_lr_params(self, lr):
+        """
+        Learning rate configuration for different layers of the model
+        Args:
+            lr (float) : Base learning rate
+        Returns:
+            list : list of parameters to optimize and their learning rates, in the format required by torch optimizers.
+        """
+        param_conf = []
+        if self.train_midas:
+            if self.encoder_lr_factor > 0:
+                param_conf.append({'params': self.core.get_enc_params_except_rel_pos(
+                ), 'lr': lr / self.encoder_lr_factor})
+
+            if self.pos_enc_lr_factor > 0:
+                param_conf.append(
+                    {'params': self.core.get_rel_pos_params(), 'lr': lr / self.pos_enc_lr_factor})
+
+            midas_params = self.core.core.scratch.parameters()
+            midas_lr_factor = self.midas_lr_factor
+            param_conf.append(
+                {'params': midas_params, 'lr': lr / midas_lr_factor})
+
+        remaining_modules = []
+        for name, child in self.named_children():
+            if name != 'core':
+                remaining_modules.append(child)
+        remaining_params = itertools.chain(
+            *[child.parameters() for child in remaining_modules])
+
+        param_conf.append({'params': remaining_params, 'lr': lr})
+
+        return param_conf
+
+    @staticmethod
+    def build(midas_model_type="DPT_BEiT_L_384", pretrained_resource=None, use_pretrained_midas=False, train_midas=False, freeze_midas_bn=True, **kwargs):
+        core = MidasCore.build(midas_model_type=midas_model_type, use_pretrained_midas=use_pretrained_midas,
+                               train_midas=train_midas, fetch_features=True, freeze_bn=freeze_midas_bn, **kwargs)
+        model = ZoeDepth(core, **kwargs)
+        if pretrained_resource:
+            assert isinstance(pretrained_resource, str), "pretrained_resource must be a string"
+            model = load_state_from_resource(model, pretrained_resource)
+        return model
+
+    @staticmethod
+    def build_from_config(config):
+        return ZoeDepth.build(**config)