Duplicate from MirageML/sjc

Co-authored-by: Aman Kishore <amankishore@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+release/diffusion_ckpts/guided_ddpm/models/lsun_bedroom.pt filter=lfs diff=lfs merge=lfs -text
+release/diffusion_ckpts/guided_ddpm/models/lsun_ffhq.pt filter=lfs diff=lfs merge=lfs -text
+release/diffusion_ckpts/stable_diffusion/sd-v1-5.ckpt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,150 @@
+*.png
+
+# sd1/
+# sd2/
+
+sde/
+
+notebooks/
+out/
+slurm_outputs/
+
+FID/torch_utils/
+FID/dnnlib/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+ckpt/
+depth/
+img/
+test*/
+view/
+vis/

LICENSE

@@ -0,0 +1,82 @@
+Copyright (c) 2022 Score Jacobian Chaining authors
+
+CreativeML Open RAIL-M
+dated August 22, 2022
+
+Section I: PREAMBLE
+
+Multimodal generative models are being widely adopted and used, and have the potential to transform the way artists, among other individuals, conceive and benefit from AI or ML technologies as a tool for content creation.
+
+Notwithstanding the current and potential benefits that these artifacts can bring to society at large, there are also concerns about potential misuses of them, either due to their technical limitations or ethical considerations.
+
+In short, this license strives for both the open and responsible downstream use of the accompanying model. When it comes to the open character, we took inspiration from open source permissive licenses regarding the grant of IP rights. Referring to the downstream responsible use, we added use-based restrictions not permitting the use of the Model in very specific scenarios, in order for the licensor to be able to enforce the license in case potential misuses of the Model may occur. At the same time, we strive to promote open and responsible research on generative models for art and content generation.
+
+Even though downstream derivative versions of the model could be released under different licensing terms, the latter will always have to include - at minimum - the same use-based restrictions as the ones in the original license (this license). We believe in the intersection between open and responsible AI development; thus, this License aims to strike a balance between both in order to enable responsible open-science in the field of AI.
+
+This License governs the use of the model (and its derivatives) and is informed by the model card associated with the model.
+
+NOW THEREFORE, You and Licensor agree as follows:
+
+1. Definitions
+
+- "License" means the terms and conditions for use, reproduction, and Distribution as defined in this document.
+- "Data" means a collection of information and/or content extracted from the dataset used with the Model, including to train, pretrain, or otherwise evaluate the Model. The Data is not licensed under this License.
+- "Output" means the results of operating a Model as embodied in informational content resulting therefrom.
+- "Model" means any accompanying machine-learning based assemblies (including checkpoints), consisting of learnt weights, parameters (including optimizer states), corresponding to the model architecture as embodied in the Complementary Material, that have been trained or tuned, in whole or in part on the Data, using the Complementary Material.
+- "Derivatives of the Model" means all modifications to the Model, works based on the Model, or any other model which is created or initialized by transfer of patterns of the weights, parameters, activations or output of the Model, to the other model, in order to cause the other model to perform similarly to the Model, including - but not limited to - distillation methods entailing the use of intermediate data representations or methods based on the generation of synthetic data by the Model for training the other model.
+- "Complementary Material" means the accompanying source code and scripts used to define, run, load, benchmark or evaluate the Model, and used to prepare data for training or evaluation, if any. This includes any accompanying documentation, tutorials, examples, etc, if any.
+- "Distribution" means any transmission, reproduction, publication or other sharing of the Model or Derivatives of the Model to a third party, including providing the Model as a hosted service made available by electronic or other remote means - e.g. API-based or web access.
+- "Licensor" means the copyright owner or entity authorized by the copyright owner that is granting the License, including the persons or entities that may have rights in the Model and/or distributing the Model.
+- "You" (or "Your") means an individual or Legal Entity exercising permissions granted by this License and/or making use of the Model for whichever purpose and in any field of use, including usage of the Model in an end-use application - e.g. chatbot, translator, image generator.
+- "Third Parties" means individuals or legal entities that are not under common control with Licensor or You.
+- "Contribution" means any work of authorship, including the original version of the Model and any modifications or additions to that Model or Derivatives of the Model thereof, that is intentionally submitted to Licensor for inclusion in the Model by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Model, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+- "Contributor" means Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Model.
+
+Section II: INTELLECTUAL PROPERTY RIGHTS
+
+Both copyright and patent grants apply to the Model, Derivatives of the Model and Complementary Material. The Model and Derivatives of the Model are subject to additional terms as described in Section III.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare, publicly display, publicly perform, sublicense, and distribute the Complementary Material, the Model, and Derivatives of the Model.
+3. Grant of Patent License. Subject to the terms and conditions of this License and where and as applicable, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this paragraph) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Model and the Complementary Material, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Model to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Model and/or Complementary Material or a Contribution incorporated within the Model and/or Complementary Material constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for the Model and/or Work shall terminate as of the date such litigation is asserted or filed.
+
+Section III: CONDITIONS OF USAGE, DISTRIBUTION AND REDISTRIBUTION
+
+4. Distribution and Redistribution. You may host for Third Party remote access purposes (e.g. software-as-a-service), reproduce and distribute copies of the Model or Derivatives of the Model thereof in any medium, with or without modifications, provided that You meet the following conditions:
+Use-based restrictions as referenced in paragraph 5 MUST be included as an enforceable provision by You in any type of legal agreement (e.g. a license) governing the use and/or distribution of the Model or Derivatives of the Model, and You shall give notice to subsequent users You Distribute to, that the Model or Derivatives of the Model are subject to paragraph 5. This provision does not apply to the use of Complementary Material.
+You must give any Third Party recipients of the Model or Derivatives of the Model a copy of this License;
+You must cause any modified files to carry prominent notices stating that You changed the files;
+You must retain all copyright, patent, trademark, and attribution notices excluding those notices that do not pertain to any part of the Model, Derivatives of the Model.
+You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions - respecting paragraph 4.a. - for use, reproduction, or Distribution of Your modifications, or for any such Derivatives of the Model as a whole, provided Your use, reproduction, and Distribution of the Model otherwise complies with the conditions stated in this License.
+5. Use-based restrictions. The restrictions set forth in Attachment A are considered Use-based restrictions. Therefore You cannot use the Model and the Derivatives of the Model for the specified restricted uses. You may use the Model subject to this License, including only for lawful purposes and in accordance with the License. Use may include creating any content with, finetuning, updating, running, training, evaluating and/or reparametrizing the Model. You shall require all of Your users who use the Model or a Derivative of the Model to comply with the terms of this paragraph (paragraph 5).
+6. The Output You Generate. Except as set forth herein, Licensor claims no rights in the Output You generate using the Model. You are accountable for the Output you generate and its subsequent uses. No use of the output can contravene any provision as stated in the License.
+
+Section IV: OTHER PROVISIONS
+
+7. Updates and Runtime Restrictions. To the maximum extent permitted by law, Licensor reserves the right to restrict (remotely or otherwise) usage of the Model in violation of this License, update the Model through electronic means, or modify the Output of the Model based on updates. You shall undertake reasonable efforts to use the latest version of the Model.
+8. Trademarks and related. Nothing in this License permits You to make use of Licensors’ trademarks, trade names, logos or to otherwise suggest endorsement or misrepresent the relationship between the parties; and any rights not expressly granted herein are reserved by the Licensors.
+9. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Model and the Complementary Material (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Model, Derivatives of the Model, and the Complementary Material and assume any risks associated with Your exercise of permissions under this License.
+10. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Model and the Complementary Material (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+11. Accepting Warranty or Additional Liability. While redistributing the Model, Derivatives of the Model and the Complementary Material thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+12. If any provision of this License is held to be invalid, illegal or unenforceable, the remaining provisions shall be unaffected thereby and remain valid as if such provision had not been set forth herein.
+
+END OF TERMS AND CONDITIONS
+
+
+
+
+Attachment A
+
+Use Restrictions
+
+You agree not to use the Model or Derivatives of the Model:
+- In any way that violates any applicable national, federal, state, local or international law or regulation;
+- For the purpose of exploiting, harming or attempting to exploit or harm minors in any way;
+- To generate or disseminate verifiably false information and/or content with the purpose of harming others;
+- To generate or disseminate personal identifiable information that can be used to harm an individual;
+- To defame, disparage or otherwise harass others;
+- For fully automated decision making that adversely impacts an individual’s legal rights or otherwise creates or modifies a binding, enforceable obligation;
+- For any use intended to or which has the effect of discriminating against or harming individuals or groups based on online or offline social behavior or known or predicted personal or personality characteristics;
+- To exploit any of the vulnerabilities of a specific group of persons based on their age, social, physical or mental characteristics, in order to materially distort the behavior of a person pertaining to that group in a manner that causes or is likely to cause that person or another person physical or psychological harm;
+- For any use intended to or which has the effect of discriminating against individuals or groups based on legally protected characteristics or categories;
+- To provide medical advice and medical results interpretation;
+- To generate or disseminate information for the purpose to be used for administration of justice, law enforcement, immigration or asylum processes, such as predicting an individual will commit fraud/crime commitment (e.g. by text profiling, drawing causal relationships between assertions made in documents, indiscriminate and arbitrarily-targeted use).

README-orig.md

@@ -0,0 +1,225 @@
+# Score Jacobian Chaining: Lifting Pretrained 2D Diffusion Models for 3D Generation
+
+[Haochen Wang*](https://whc.is/),
+[Xiaodan Du*](https://github.com/duxiaodan),
+[Jiahao Li*](https://www.linkedin.com/in/jiahaoli95/),
+[Raymond A. Yeh†](https://raymond-yeh.com),
+[Greg Shakhnarovich](https://home.ttic.edu/~gregory/)
+(* indicates equal contribution)
+
+TTI-Chicago, †Purdue University
+
+Abstract: *A diffusion model learns to predict a vector field of gradients. We propose to apply chain rule on the learned gradients, and back-propagate the score of a diffusion model through the Jacobian of a differentiable renderer, which we instantiate to be a voxel radiance field. This setup aggregates 2D scores at multiple camera viewpoints into a 3D score, and repurposes a pretrained 2D model for 3D data generation. We identify a technical challenge of distribution mismatch that arises in this application, and propose a novel estimation mechanism to resolve it. We run our algorithm on several off-the-shelf diffusion image generative models, including the recently released Stable Diffusion trained on the large-scale LAION dataset.*
+
+
+<a href="https://arxiv.org/abs/2212.00774"><img src="https://img.shields.io/badge/arXiv-2212.00774-b31b1b.svg" height=22.5></a>
+<a href="https://colab.research.google.com/drive/1zixo66UYGl70VOPy053o7IV_YkQt5lCZ?usp=sharing"><img src="https://colab.research.google.com/assets/colab-badge.svg" height=22.5></a>
+<a href="https://pals.ttic.edu/p/score-jacobian-chaining"><img src="https://img.shields.io/website?down_color=lightgrey&down_message=offline&label=Project%20Page&up_color=lightgreen&up_message=online&url=https%3A%2F%2Fpals.ttic.edu%2Fp%2Fscore-jacobian-chaining" height=22.5></a>
+
+<!-- [ [arxiv](https://arxiv.org/abs/2212.00774) | [project page](https://pals.ttic.edu/p/score-jacobian-chaining) | [colab](https://colab.research.google.com/drive/1zixo66UYGl70VOPy053o7IV_YkQt5lCZ?usp=sharing ) ] -->
+
+Many thanks to [dvschultz](https://github.com/dvschultz) for the colab.
+
+## Updates
+- We have added subpixel rendering script for final high quality vis. The jittery videos you might have seen should be significantly better now. Please run `python /path/to/sjc/highres_final_vis.py` in the exp folder after the training is complete. There are a few toggles in the script you can play with, but the default is ok. It takes about 5 minutes / 11GB on an A5000, and the extra time is mainly due to SD Decoder.
+- If you are running SJC with a DreamBooth fine-tuned model: the model's output distribution is already significantly narrowed. It might help to use a lower guidance scale `--sd.scale 50.0` for example. Intense mode-seeking is one cause for multi-face problem. We have internally tried DreamBooth with view-dependent prompt fine-tuning. But by and large DreamBooth integration is not ready.
+
+
+## TODOs
+- [ ] make seeds configurable. So far all seeds are hardcoded to 0.
+- [ ] add script to reproduce 2D experiments in Fig 4. The Fig might need change once it's tied to seeds. Note that for a simple aligned domain like faces, simple scheduling like using a single σ=1.5 could already generate some nice images. But not so for bedrooms; it's too diverse and annealing seems still needed.
+- [ ] main paper figures did not use subpix rendering; appendix figures did. Replace the main paper figures to make them consistent.
+
+## License
+Since we use Stable Diffusion, we are releasing under their OpenRAIL license. Otherwise we do not
+identify any components or upstream code that carry restrictive licensing requirements.
+
+## Structure
+In addition to SJC, the repo also contains an implementation of [Karras sampler](https://arxiv.org/abs/2206.00364),
+and a customized, simple voxel nerf. We provide the abstract parent class based on Karras et. al. and include
+a few types of diffusion model here. See adapt.py.
+
+## Installation
+
+Install Pytorch according to your CUDA version, for example:
+```bash
+pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu116
+```
+
+Install other dependencies by `pip install -r requirements.txt`.
+
+Install `taming-transformers` manually
+```bash
+git clone --depth 1 git@github.com:CompVis/taming-transformers.git && pip install -e taming-transformers
+```
+
+## Downloading checkpoints
+We have bundled a minimal set of things you need to download (SD v1.5 ckpt, gddpm ckpt for LSUN and FFHQ)
+in a tar file, made available at our download server [here](https://dl.ttic.edu/pals/sjc/release.tar).
+It is a single file of 12GB, and you can use wget or curl.
+
+Remember to __update__ `env.json` to point at the new checkpoint root where you have uncompressed the files.
+
+## Usage
+Make a new directory to run experiments (the script generates many logging files. Do not run at the root of the code repo, else risk contamination.)
+```bash
+mkdir exp
+cd exp
+```
+Run the following command to generate a new 3D asset. It takes about 25 minutes / 10GB GPU mem on a single A5000 GPU for 10000 steps of optimization.
+```bash
+python /path/to/sjc/run_sjc.py \
+--sd.prompt "A zoomed out high quality photo of Temple of Heaven" \
+--n_steps 10000 \
+--lr 0.05 \
+--sd.scale 100.0 \
+--emptiness_weight 10000 \
+--emptiness_step 0.5 \
+--emptiness_multiplier 20.0 \
+--depth_weight 0 \
+--var_red False
+```
+`sd.prompt` is the prompt to the stable diffusion model
+
+`n_steps` is the number of gradient steps
+
+`lr` is the base learning rate of the optimizer
+
+`sd.scale` is the guidance scale for stable diffusion
+
+`emptiness_weight` is the weighting factor of the emptiness loss
+
+`emptiness_step` indicates after `emptiness_step * n_steps` update steps, the `emptiness_weight` is multiplied by `emptiness_multiplier`.
+
+`emptiness_multipler` see above
+
+`depth_weight` the weighting factor of the center depth loss
+
+`var_red` whether to use Eq. 16 vs Eq. 15. For some prompts such as Obama we actually see better results with Eq. 15.
+
+Visualization results are stored in the current directory. In directories named `test_*` there are images (under `view`) and videos (under `view_seq`) rendered at different iterations.
+
+
+## To Reproduce the Results in the Paper
+First create a clean directory for your experiment, then run one of the following scripts from that folder:
+### Trump
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "Trump figure" --n_steps 30000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Obama
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "Obama figure" --n_steps 30000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Biden
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "Biden figure" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Temple of Heaven
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A zoomed out high quality photo of Temple of Heaven" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Burger
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A high quality photo of a delicious burger" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Icecream
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A high quality photo of a chocolate icecream cone" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 10
+
+```
+### Ficus
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A ficus planted in a pot" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 100
+```
+### Castle
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A zoomed out photo a small castle" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 50
+```
+### Sydney Opera House
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A zoomed out high quality photo of Sydney Opera House" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Rose
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "a DSLR photo of a rose" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 50
+```
+### School Bus
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A high quality photo of a yellow school bus" --n_steps 30000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0 --var_red False
+```
+### Rocket
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A wide angle zoomed out photo of Saturn V rocket from distance" --n_steps 30000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0  --var_red False
+```
+### French Fries
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A high quality photo of french fries from McDonald's" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 10
+```
+### Motorcycle
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A high quality photo of a toy motorcycle" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Car
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A high quality photo of a classic silver muscle car" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Tank
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A product photo of a toy tank" --n_steps 20000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Chair
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A high quality photo of a Victorian style wooden chair with velvet upholstery" --n_steps 50000 --lr 0.01 --sd.scale 100.0 --emptiness_weight 7000
+```
+### Duck
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "a DSLR photo of a yellow duck" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 10
+```
+### Horse
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A photo of a horse walking" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+### Giraffe
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A wide angle zoomed out photo of a giraffe" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 50
+```
+### Zebra
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A photo of a zebra walking" --n_steps 10000 --lr 0.02 --sd.scale 100.0 --emptiness_weight 30000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0 --var_red False
+```
+### Printer
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A product photo of a Canon home printer" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0 --var_red False
+```
+### Zelda Link
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "Zelda Link" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0 --var_red False
+```
+### Pig
+```
+python /path/to/sjc/run_sjc.py --sd.prompt "A pig" --n_steps 10000 --lr 0.05 --sd.scale 100.0 --emptiness_weight 10000 --emptiness_step 0.5 --emptiness_multiplier 20.0 --depth_weight 0
+```
+
+
+## To Test the Voxel NeRF
+```
+python /path/to/sjc/run_nerf.py
+```
+Our bundle contains a tar ball for the lego bulldozer dataset. Untar it and it will work.
+
+## To Sample 2D images with the Karras Sampler
+```
+python /path/to/sjc/run_img_sampling.py
+```
+Use help -h to see the options available. Will expand the details later.
+
+
+## Bib
+```
+@article{sjc,
+      title={Score Jacobian Chaining: Lifting Pretrained 2D Diffusion Models for 3D Generation},
+      author={Wang, Haochen and Du, Xiaodan and Li, Jiahao and Yeh, Raymond A. and Shakhnarovich, Greg},
+      journal={arXiv preprint arXiv:2212.00774},
+      year={2022},
+}
+```

README.md

@@ -0,0 +1,22 @@
+---
+title: Score Jacobian Chaining
+emoji: 🧊
+colorFrom: red
+colorTo: pink
+sdk: gradio
+sdk_version: 3.12.0
+app_file: app.py
+pinned: false
+license: creativeml-openrail-m
+duplicated_from: MirageML/sjc
+---
+
+## Bib
+```
+@article{sjc,
+      title={Score Jacobian Chaining: Lifting Pretrained 2D Diffusion Models for 3D Generation},
+      author={Wang, Haochen and Du, Xiaodan and Li, Jiahao and Yeh, Raymond A. and Shakhnarovich, Greg},
+      journal={arXiv preprint arXiv:2212.00774},
+      year={2022},
+}
+```

adapt.py

@@ -0,0 +1,163 @@
+from pathlib import Path
+import json
+from math import sqrt
+import numpy as np
+import torch
+from abc import ABCMeta, abstractmethod
+
+
+class ScoreAdapter(metaclass=ABCMeta):
+
+    @abstractmethod
+    def denoise(self, xs, σ, **kwargs):
+        pass
+
+    def score(self, xs, σ, **kwargs):
+        Ds = self.denoise(xs, σ, **kwargs)
+        grad_log_p_t = (Ds - xs) / (σ ** 2)
+        return grad_log_p_t
+
+    @abstractmethod
+    def data_shape(self):
+        return (3, 256, 256)  # for example
+
+    def samps_centered(self):
+        # if centered, samples expected to be in range [-1, 1], else [0, 1]
+        return True
+
+    @property
+    @abstractmethod
+    def σ_max(self):
+        pass
+
+    @property
+    @abstractmethod
+    def σ_min(self):
+        pass
+
+    def cond_info(self, batch_size):
+        return {}
+
+    @abstractmethod
+    def unet_is_cond(self):
+        return False
+
+    @abstractmethod
+    def use_cls_guidance(self):
+        return False  # most models do not use cls guidance
+
+    def classifier_grad(self, xs, σ, ys):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def snap_t_to_nearest_tick(self, t):
+        # need to confirm for each model; continuous time model doesn't need this
+        return t, None
+
+    @property
+    def device(self):
+        return self._device
+
+    def checkpoint_root(self):
+        """the path at which the pretrained checkpoints are stored"""
+        with Path(__file__).resolve().with_name("env.json").open("r") as f:
+            root = json.load(f)['data_root']
+            root = Path(root) / "diffusion_ckpts"
+        return root
+
+
+def karras_t_schedule(ρ=7, N=10, σ_max=80, σ_min=0.002):
+    ts = []
+    for i in range(N):
+
+        t = (
+            σ_max ** (1 / ρ) + (i / (N - 1)) * (σ_min ** (1 / ρ) - σ_max ** (1 / ρ))
+        ) ** ρ
+        ts.append(t)
+    return ts
+
+
+def power_schedule(σ_max, σ_min, num_stages):
+    σs = np.exp(np.linspace(np.log(σ_max), np.log(σ_min), num_stages))
+    return σs
+
+
+class Karras():
+
+    @classmethod
+    @torch.no_grad()
+    def inference(
+        cls, model, batch_size, num_t, *,
+        σ_max=80, cls_scaling=1,
+        init_xs=None, heun=True,
+        langevin=False,
+        S_churn=80, S_min=0.05, S_max=50, S_noise=1.003,
+    ):
+        σ_max = min(σ_max, model.σ_max)
+        σ_min = model.σ_min
+        ts = karras_t_schedule(ρ=7, N=num_t, σ_max=σ_max, σ_min=σ_min)
+        assert len(ts) == num_t
+        ts = [model.snap_t_to_nearest_tick(t)[0] for t in ts]
+        ts.append(0)  # 0 is the destination
+        σ_max = ts[0]
+
+        cond_inputs = model.cond_info(batch_size)
+
+        def compute_step(xs, σ):
+            grad_log_p_t = model.score(
+                xs, σ, **(cond_inputs if model.unet_is_cond() else {})
+            )
+            if model.use_cls_guidance():
+                grad_cls = model.classifier_grad(xs, σ, cond_inputs["y"])
+                grad_cls = grad_cls * cls_scaling
+                grad_log_p_t += grad_cls
+            d_i = -1 * σ * grad_log_p_t
+            return d_i
+
+        if init_xs is not None:
+            xs = init_xs.to(model.device)
+        else:
+            xs = σ_max * torch.randn(
+                batch_size, *model.data_shape(), device=model.device
+            )
+
+        yield xs
+
+        for i in range(num_t):
+            t_i = ts[i]
+
+            if langevin and (S_min < t_i and t_i < S_max):
+                xs, t_i = cls.noise_backward_in_time(
+                    model, xs, t_i, S_noise, S_churn / num_t
+                )
+
+            Δt = ts[i+1] - t_i
+
+            d_1 = compute_step(xs, σ=t_i)
+            xs_1 = xs + Δt * d_1
+
+            # Heun's 2nd order method; don't apply on the last step
+            if (not heun) or (ts[i+1] == 0):
+                xs = xs_1
+            else:
+                d_2 = compute_step(xs_1, σ=ts[i+1])
+                xs = xs + Δt * (d_1 + d_2) / 2
+
+            yield xs
+
+    @staticmethod
+    def noise_backward_in_time(model, xs, t_i, S_noise, S_churn_i):
+        n = S_noise * torch.randn_like(xs)
+        γ_i = min(sqrt(2)-1, S_churn_i)
+        t_i_hat = t_i * (1 + γ_i)
+        t_i_hat = model.snap_t_to_nearest_tick(t_i_hat)[0]
+        xs = xs + n * sqrt(t_i_hat ** 2 - t_i ** 2)
+        return xs, t_i_hat
+
+
+def test():
+    pass
+
+
+if __name__ == "__main__":
+    test()

adapt_gddpm.py

@@ -0,0 +1,562 @@
+from pathlib import Path
+from math import sin, pi, sqrt
+from functools import partial
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from easydict import EasyDict
+from guided_diffusion.script_util import (
+    create_model_and_diffusion,
+    model_and_diffusion_defaults,
+
+    NUM_CLASSES,
+    create_classifier,
+    classifier_defaults,
+
+    sr_create_model_and_diffusion,
+    sr_model_and_diffusion_defaults,
+)
+
+from adapt import ScoreAdapter
+
+from my.registry import Registry
+
+PRETRAINED_REGISTRY = Registry("pretrained")
+
+
+device = torch.device("cuda")
+
+
+def load_ckpt(path, **kwargs):
+    # with bf.BlobFile(path, "rb") as f:
+    #     data = f.read()
+    return torch.load(path, **kwargs)
+
+
+def pick_out_cfgs(src, target_ks):
+    return {k: src[k] for k in target_ks}
+
+
+@PRETRAINED_REGISTRY.register()
+def m_imgnet_64():
+    return dict(
+        attention_resolutions="32,16,8",
+        class_cond=True,
+        diffusion_steps=1000,
+        dropout=0.1,
+        image_size=64,
+        learn_sigma=True,
+        noise_schedule="cosine",
+        num_channels=192,
+        num_head_channels=64,
+        num_res_blocks=3,
+        resblock_updown=True,
+        use_new_attention_order=True,
+        use_fp16=True,
+        use_scale_shift_norm=True,
+
+        classifier_depth=4,
+
+        classifier_scale=1.0,
+        model_path="models/64x64_diffusion.pt",
+        classifier_path="models/64x64_classifier.pt",
+    )
+
+
+@PRETRAINED_REGISTRY.register()
+def m_imgnet_128():
+    return dict(
+        attention_resolutions="32,16,8",
+        class_cond=True,
+        diffusion_steps=1000,
+        image_size=128,
+        learn_sigma=True,
+        noise_schedule="linear",
+        num_channels=256,
+        num_heads=4,
+        num_res_blocks=2,
+        resblock_updown=True,
+        use_fp16=True,
+        use_scale_shift_norm=True,
+
+        classifier_scale=0.5,
+        model_path="models/128x128_diffusion.pt",
+        classifier_path="models/128x128_classifier.pt",
+    )
+
+
+@PRETRAINED_REGISTRY.register()
+def m_imgnet_256():
+    return dict(
+        attention_resolutions="32,16,8",
+        class_cond=True,
+        diffusion_steps=1000,
+        image_size=256,
+        learn_sigma=True,
+        noise_schedule="linear",
+        num_channels=256,
+        num_head_channels=64,
+        num_res_blocks=2,
+        resblock_updown=True,
+        use_fp16=True,
+        use_scale_shift_norm=True,
+
+        classifier_scale=1.0,
+        model_path="models/256x256_diffusion.pt",
+        classifier_path="models/256x256_classifier.pt"
+    )
+
+
+@PRETRAINED_REGISTRY.register()
+def m_imgnet_256_uncond():
+    return dict(
+        attention_resolutions="32,16,8",
+        class_cond=False,
+        diffusion_steps=1000,
+        image_size=256,
+        learn_sigma=True,
+        noise_schedule="linear",
+        num_channels=256,
+        num_head_channels=64,
+        num_res_blocks=2,
+        resblock_updown=True,
+        use_fp16=True,
+        use_scale_shift_norm=True,
+
+        classifier_scale=10.0,
+        model_path="models/256x256_diffusion_uncond.pt",
+        classifier_path="models/256x256_classifier.pt",
+    )
+
+
+@PRETRAINED_REGISTRY.register()
+def m_imgnet_512():
+    return dict(
+        attention_resolutions="32,16,8",
+        class_cond=True,
+        diffusion_steps=1000,
+        image_size=512,
+        learn_sigma=True,
+        noise_schedule="linear",
+        num_channels=256,
+        num_head_channels=64,
+        num_res_blocks=2,
+        resblock_updown=True,
+        use_fp16=False,
+        use_scale_shift_norm=True,
+
+        classifier_scale=4.0,
+        model_path="models/512x512_diffusion.pt",
+        classifier_path="models/512x512_classifier.pt"
+    )
+
+
+@PRETRAINED_REGISTRY.register()
+def m_imgnet_64_256(base_samples="64_samples.npz"):
+    return dict(
+        attention_resolutions="32,16,8",
+        class_cond=True,
+        diffusion_steps=1000,
+        large_size=256,
+        small_size=64,
+        learn_sigma=True,
+        noise_schedule="linear",
+        num_channels=192,
+        num_heads=4,
+        num_res_blocks=2,
+        resblock_updown=True,
+        use_fp16=True,
+        use_scale_shift_norm=True,
+
+        model_path="models/64_256_upsampler.pt",
+
+        base_samples=base_samples,
+    )
+
+
+@PRETRAINED_REGISTRY.register()
+def m_imgnet_128_512(base_samples="128_samples.npz",):
+    return dict(
+        attention_resolutions="32,16",
+        class_cond=True,
+        diffusion_steps=1000,
+        large_size=512,
+        small_size=128,
+        learn_sigma=True,
+        noise_schedule="linear",
+        num_channels=192,
+        num_head_channels=64,
+        num_res_blocks=2,
+        resblock_updown=True,
+        use_fp16=True,
+        use_scale_shift_norm=True,
+
+        model_path="models/128_512_upsampler.pt",
+
+        base_samples=base_samples,
+    )
+
+
+@PRETRAINED_REGISTRY.register()
+def m_lsun_256(category="bedroom"):
+    return dict(
+        attention_resolutions="32,16,8",
+        class_cond=False,
+        diffusion_steps=1000,
+        dropout=0.1,
+        image_size=256,
+        learn_sigma=True,
+        noise_schedule="linear",
+        num_channels=256,
+        num_head_channels=64,
+        num_res_blocks=2,
+        resblock_updown=True,
+        use_fp16=True,
+        use_scale_shift_norm=True,
+
+        model_path=f"models/lsun_{category}.pt"
+    )
+
+
+def img_gen(specific_cfgs, num_samples=16, batch_size=16, load_only=False, ckpt_root=Path("")):
+    cfgs = EasyDict(
+        clip_denoised=True,
+        num_samples=num_samples,
+        batch_size=batch_size,
+        use_ddim=False,
+        model_path="",
+        classifier_path="",
+        classifier_scale=1.0,
+    )
+    cfgs.update(model_and_diffusion_defaults())
+    cfgs.update(classifier_defaults())
+    cfgs.update(specific_cfgs)
+
+    use_classifier_guidance = bool(cfgs.classifier_path)
+    class_aware = cfgs.class_cond or use_classifier_guidance
+
+    model, diffusion = create_model_and_diffusion(
+        **pick_out_cfgs(cfgs, model_and_diffusion_defaults().keys())
+    )
+    model.load_state_dict(
+        load_ckpt(str(ckpt_root / cfgs.model_path), map_location="cpu")
+    )
+    model.to(device)
+    if cfgs.use_fp16:
+        model.convert_to_fp16()
+    model.eval()
+
+    def model_fn(x, t, y=None):
+        return model(x, t, y if cfgs.class_cond else None)
+
+    classifier = None
+    cond_fn = None
+    if use_classifier_guidance:
+        classifier = create_classifier(
+            **pick_out_cfgs(cfgs, classifier_defaults().keys())
+        )
+        classifier.load_state_dict(
+            load_ckpt(str(ckpt_root / cfgs.classifier_path), map_location="cpu")
+        )
+        classifier.to(device)
+        if cfgs.classifier_use_fp16:
+            classifier.convert_to_fp16()
+        classifier.eval()
+
+        def cond_fn(x, t, y=None):
+            assert y is not None
+            with torch.enable_grad():
+                x_in = x.detach().requires_grad_(True)
+                logits = classifier(x_in, t)
+                log_probs = F.log_softmax(logits, dim=-1)
+                selected = log_probs[range(len(logits)), y.view(-1)]
+                return torch.autograd.grad(selected.sum(), x_in)[0] * cfgs.classifier_scale
+
+    if load_only:
+        return model, classifier
+
+    all_images = []
+    all_labels = []
+
+    while len(all_images) * cfgs.batch_size < cfgs.num_samples:
+        model_kwargs = {}
+
+        if class_aware:
+            classes = torch.randint(
+                low=0, high=NUM_CLASSES, size=(cfgs.batch_size,), device=device
+            )
+            model_kwargs["y"] = classes
+
+        sample_fn = (
+            diffusion.p_sample_loop if not cfgs.use_ddim else diffusion.ddim_sample_loop
+        )
+        sample = sample_fn(
+            model_fn,
+            (cfgs.batch_size, 3, cfgs.image_size, cfgs.image_size),
+            clip_denoised=cfgs.clip_denoised,
+            model_kwargs=model_kwargs,
+            cond_fn=cond_fn,
+            device=device,
+            progress=True
+        )
+        sample = ((sample + 1) * 127.5).clamp(0, 255).to(torch.uint8)
+        sample = sample.permute(0, 2, 3, 1)
+        sample = sample.contiguous()
+
+        all_images.append(sample.cpu().numpy())
+        if class_aware:
+            all_labels.append(classes.cpu().numpy())
+
+    arr = np.concatenate(all_images, axis=0)
+    arr = arr[:cfgs.num_samples]
+
+    if class_aware:
+        all_labels = np.concatenate(all_labels, axis=0)
+        all_labels = all_labels[:cfgs.num_samples]
+
+    shape_str = "x".join([str(x) for x in arr.shape])
+    out_path = Path("./out") / f"samples_{shape_str}.npz"
+    np.savez(out_path, arr, all_labels)
+
+
+def img_upsamp(specific_cfgs, num_samples=16, batch_size=16, load_only=False):
+    """note that here the ckpt root is not configured properly; will break but easy fix"""
+    cfgs = EasyDict(
+        clip_denoised=True,
+        num_samples=num_samples,
+        batch_size=batch_size,
+        use_ddim=False,
+        base_samples="",
+        model_path="",
+    )
+    cfgs.update(sr_model_and_diffusion_defaults())
+    cfgs.update(specific_cfgs)
+
+    model, diffusion = sr_create_model_and_diffusion(
+        **pick_out_cfgs(cfgs, sr_model_and_diffusion_defaults().keys())
+    )
+    model.load_state_dict(load_ckpt(cfgs.model_path, map_location="cpu"))
+    model.to(device)
+    if cfgs.use_fp16:
+        model.convert_to_fp16()
+    model.eval()
+
+    if load_only:
+        return model
+
+    data = load_low_res_samples(
+        cfgs.base_samples, cfgs.batch_size, cfgs.class_cond
+    )
+
+    all_images = []
+    while len(all_images) * cfgs.batch_size < cfgs.num_samples:
+        model_kwargs = next(data)
+        model_kwargs = {k: v.to(device) for k, v in model_kwargs.items()}
+        samples = diffusion.p_sample_loop(
+            model,
+            (cfgs.batch_size, 3, cfgs.large_size, cfgs.large_size),
+            clip_denoised=cfgs.clip_denoised,
+            model_kwargs=model_kwargs,
+            progress=True
+        )
+        samples = ((samples + 1) * 127.5).clamp(0, 255).to(torch.uint8)
+        samples = samples.permute(0, 2, 3, 1)
+        samples = samples.contiguous()
+
+        all_images.append(samples.cpu().numpy())
+
+    arr = np.concatenate(all_images, axis=0)
+    arr = arr[: cfgs.num_samples]
+
+    shape_str = "x".join([str(x) for x in arr.shape])
+    out_path = Path("./out") / f"samples_{shape_str}.npz"
+    np.savez(out_path, arr)
+
+
+def load_low_res_samples(base_samples, batch_size, class_cond):
+    obj = np.load(base_samples)
+    image_arr = obj["arr_0"]
+    if class_cond:
+        label_arr = obj["arr_1"]
+
+    buffer = []
+    label_buffer = []
+    while True:
+        for i in range(len(image_arr)):
+            buffer.append(image_arr[i])
+            if class_cond:
+                label_buffer.append(label_arr[i])
+
+            if len(buffer) == batch_size:
+                batch = torch.from_numpy(np.stack(buffer)).float()
+                batch = batch / 127.5 - 1.0
+                batch = batch.permute(0, 3, 1, 2)
+                res = {}
+                res["low_res"] = batch
+                if class_cond:
+                    res["y"] = torch.from_numpy(np.stack(label_buffer))
+                yield res
+                buffer, label_buffer = [], []
+
+
+def class_cond_info(imgnet_cat):
+
+    def rand_cond_fn(batch_size):
+        cats = torch.randint(
+            low=0, high=NUM_CLASSES, size=(batch_size,), device=device
+        )
+        return {"y": cats}
+
+    def class_specific_cond(batch_size):
+        cats = torch.tensor([imgnet_cat, ] * batch_size, device=device)
+        return {"y": cats}
+
+    if imgnet_cat == -1:
+        return rand_cond_fn
+    else:
+        return class_specific_cond
+
+
+def _sqrt(x):
+    if isinstance(x, float):
+        return sqrt(x)
+    else:
+        assert isinstance(x, torch.Tensor)
+        return torch.sqrt(x)
+
+
+class GuidedDDPM(ScoreAdapter):
+    def __init__(self, model, lsun_cat, imgnet_cat):
+        print(PRETRAINED_REGISTRY)
+        cfgs = PRETRAINED_REGISTRY.get(model)(
+            **({"category": lsun_cat} if model.startswith("m_lsun") else {})
+        )
+
+        self.unet, self.classifier = img_gen(
+            cfgs, load_only=True, ckpt_root=self.checkpoint_root() / "guided_ddpm"
+        )
+
+        H, W = cfgs['image_size'], cfgs['image_size']
+        self._data_shape = (3, H, W)
+
+        if cfgs['class_cond'] or (self.classifier is not None):
+            cond_func = class_cond_info(imgnet_cat)
+        else:
+            cond_func = lambda *args, **kwargs: {}
+        self.cond_func = cond_func
+
+        self._unet_is_cond = bool(cfgs['class_cond'])
+
+        noise_schedule = cfgs['noise_schedule']
+        assert noise_schedule in ("linear", "cosine")
+        self.M = 1000
+        if noise_schedule == "linear":
+            self.us = self.linear_us(self.M)
+            self._σ_min = 0.01
+        else:
+            self.us = self.cosine_us(self.M)
+            self._σ_min = 0.0064
+        self.noise_schedule = noise_schedule
+
+        self._device = next(self.unet.parameters()).device
+
+    def data_shape(self):
+        return self._data_shape
+
+    @property
+    def σ_max(self):
+        return self.us[0]
+
+    @property
+    def σ_min(self):
+        return self.us[-1]
+
+    @torch.no_grad()
+    def denoise(self, xs, σ, **model_kwargs):
+        N = xs.shape[0]
+        cond_t, σ = self.time_cond_vec(N, σ)
+        output = self.unet(
+            xs / _sqrt(1 + σ**2), cond_t, **model_kwargs
+        )
+        # not using the var pred
+        n_hat = torch.split(output, xs.shape[1], dim=1)[0]
+        Ds = xs - σ * n_hat
+        return Ds
+
+    def cond_info(self, batch_size):
+        return self.cond_func(batch_size)
+
+    def unet_is_cond(self):
+        return self._unet_is_cond
+
+    def use_cls_guidance(self):
+        return (self.classifier is not None)
+
+    @torch.no_grad()
+    def classifier_grad(self, xs, σ, ys):
+        N = xs.shape[0]
+        cond_t, σ = self.time_cond_vec(N, σ)
+        with torch.enable_grad():
+            x_in = xs.detach().requires_grad_(True)
+            logits = self.classifier(x_in, cond_t)
+            log_probs = F.log_softmax(logits, dim=-1)
+            selected = log_probs[range(len(logits)), ys.view(-1)]
+            grad = torch.autograd.grad(selected.sum(), x_in)[0]
+
+        grad = grad * (1 / sqrt(1 + σ**2))
+        return grad
+
+    def snap_t_to_nearest_tick(self, t):
+        j = np.abs(t - self.us).argmin()
+        return self.us[j], j
+
+    def time_cond_vec(self, N, σ):
+        if isinstance(σ, float):
+            σ, j = self.snap_t_to_nearest_tick(σ)  # σ might change due to snapping
+            cond_t = (self.M - 1) - j
+            cond_t = torch.tensor([cond_t] * N, device=self.device)
+            return cond_t, σ
+        else:
+            assert isinstance(σ, torch.Tensor)
+            σ = σ.reshape(-1).cpu().numpy()
+            σs = []
+            js = []
+            for elem in σ:
+                _σ, _j = self.snap_t_to_nearest_tick(elem)
+                σs.append(_σ)
+                js.append((self.M - 1) - _j)
+
+            cond_t = torch.tensor(js, device=self.device)
+            σs = torch.tensor(σs, device=self.device, dtype=torch.float32).reshape(-1, 1, 1, 1)
+            return cond_t, σs
+
+    @staticmethod
+    def cosine_us(M=1000):
+        assert M == 1000
+
+        def α_bar(j):
+            return sin(pi / 2 * j / (M * (0.008 + 1))) ** 2
+
+        us = [0, ]
+        for j in reversed(range(0, M)):  # [M-1, 0], inclusive
+            u_j = sqrt(((us[-1] ** 2) + 1) / (max(α_bar(j) / α_bar(j+1), 0.001)) - 1)
+            us.append(u_j)
+
+        us = np.array(us)
+        us = us[1:]
+        us = us[::-1]
+        return us
+
+    @staticmethod
+    def linear_us(M=1000):
+        assert M == 1000
+        β_start = 0.0001
+        β_end = 0.02
+        βs = np.linspace(β_start, β_end, M, dtype=np.float64)
+        αs = np.cumprod(1 - βs)
+        us = np.sqrt((1 - αs) / αs)
+        us = us[::-1]
+        return us

adapt_ncsn.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+import argparse
+import yaml
+
+import numpy as np
+import torch
+
+from ncsn.ncsnv2 import NCSNv2, NCSNv2Deeper, NCSNv2Deepest, get_sigmas
+from ncsn.ema import EMAHelper
+
+from adapt import ScoreAdapter
+
+device = torch.device("cuda")
+
+
+def get_model(config):
+    if config.data.dataset == 'CIFAR10' or config.data.dataset == 'CELEBA':
+        return NCSNv2(config).to(config.device)
+    elif config.data.dataset == "FFHQ":
+        return NCSNv2Deepest(config).to(config.device)
+    elif config.data.dataset == 'LSUN':
+        return NCSNv2Deeper(config).to(config.device)
+
+
+def dict2namespace(config):
+    namespace = argparse.Namespace()
+    for key, value in config.items():
+        if isinstance(value, dict):
+            new_value = dict2namespace(value)
+        else:
+            new_value = value
+        setattr(namespace, key, new_value)
+    return namespace
+
+
+class NCSN(ScoreAdapter):
+    def __init__(self):
+        config_fname = Path(__file__).resolve().parent / "ncsn" / "bedroom.yml"
+        with config_fname.open("r") as f:
+            config = yaml.safe_load(f)
+            config = dict2namespace(config)
+
+        config.device = device
+
+        states = torch.load(
+            self.checkpoint_root() / "ncsn/exp/logs/bedroom/checkpoint_150000.pth"
+        )
+
+        model = get_model(config)
+        model = torch.nn.DataParallel(model)
+        model.load_state_dict(states[0], strict=True)
+
+        if config.model.ema:
+            ema_helper = EMAHelper(mu=config.model.ema_rate)
+            ema_helper.register(model)
+            ema_helper.load_state_dict(states[-1])
+            # HC: update the model param with history ema.
+            # if don't do this the colors of images become strangely saturated.
+            # this is reported in the paper.
+            ema_helper.ema(model)
+
+        model = model.module  # remove DataParallel
+        model.eval()
+        self.model = model
+        self._data_shape = (3, config.data.image_size, config.data.image_size)
+
+        self.σs = model.sigmas.cpu().numpy()
+        self._device = device
+
+    def data_shape(self):
+        return self._data_shape
+
+    def samps_centered(self):
+        return False
+
+    @property
+    def σ_max(self):
+        return self.σs[0]
+
+    @property
+    def σ_min(self):
+        return self.σs[-1]
+
+    @torch.no_grad()
+    def denoise(self, xs, σ):
+        σ, j = self.snap_t_to_nearest_tick(σ)
+        N = xs.shape[0]
+        cond_t = torch.tensor([j] * N, dtype=torch.long, device=self.device)
+        score = self.model(xs, cond_t)
+        Ds = xs + score * (σ ** 2)
+        return Ds
+
+    def unet_is_cond(self):
+        return False
+
+    def use_cls_guidance(self):
+        return False
+
+    def snap_t_to_nearest_tick(self, t):
+        j = np.abs(t - self.σs).argmin()
+        return self.σs[j], j

adapt_sd.py

@@ -0,0 +1,235 @@
+import sys
+from pathlib import Path
+import torch
+import numpy as np
+from omegaconf import OmegaConf
+from einops import rearrange
+
+from torch import autocast
+from contextlib import nullcontext
+from math import sqrt
+from adapt import ScoreAdapter
+
+import warnings
+from transformers import logging
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+logging.set_verbosity_error()
+
+
+device = torch.device("cuda")
+
+
+def curr_dir():
+    return Path(__file__).resolve().parent
+
+
+def add_import_path(dirname):
+    sys.path.append(str(
+        curr_dir() / str(dirname)
+    ))
+
+
+def load_model_from_config(config, ckpt, verbose=False):
+    from ldm.util import instantiate_from_config
+    print(f"Loading model from {ckpt}")
+    pl_sd = torch.load(ckpt, map_location="cpu")
+    if "global_step" in pl_sd:
+        print(f"Global Step: {pl_sd['global_step']}")
+    sd = pl_sd["state_dict"]
+    model = instantiate_from_config(config.model)
+    m, u = model.load_state_dict(sd, strict=False)
+    if len(m) > 0 and verbose:
+        print("missing keys:")
+        print(m)
+    if len(u) > 0 and verbose:
+        print("unexpected keys:")
+        print(u)
+
+    model.to(device)
+    model.eval()
+    return model
+
+
+def load_sd1_model(ckpt_root):
+    ckpt_fname = ckpt_root / "stable_diffusion" / "sd-v1-5.ckpt"
+    cfg_fname = curr_dir() / "sd1" / "configs" / "v1-inference.yaml"
+    H, W = 512, 512
+
+    config = OmegaConf.load(str(cfg_fname))
+    model = load_model_from_config(config, str(ckpt_fname))
+    return model, H, W
+
+
+def load_sd2_model(ckpt_root, v2_highres):
+    if v2_highres:
+        ckpt_fname = ckpt_root / "sd2" / "768-v-ema.ckpt"
+        cfg_fname = curr_dir() / "sd2/configs/stable-diffusion/v2-inference-v.yaml"
+        H, W = 768, 768
+    else:
+        ckpt_fname = ckpt_root / "sd2" / "512-base-ema.ckpt"
+        cfg_fname = curr_dir() / "sd2/configs/stable-diffusion/v2-inference.yaml"
+        H, W = 512, 512
+
+    config = OmegaConf.load(f"{cfg_fname}")
+    model = load_model_from_config(config, str(ckpt_fname))
+    return model, H, W
+
+
+def _sqrt(x):
+    if isinstance(x, float):
+        return sqrt(x)
+    else:
+        assert isinstance(x, torch.Tensor)
+        return torch.sqrt(x)
+
+
+class StableDiffusion(ScoreAdapter):
+    def __init__(self, variant, v2_highres, prompt, scale, precision):
+        if variant == "v1":
+            add_import_path("sd1")
+            self.model, H, W = load_sd1_model(self.checkpoint_root())
+        elif variant == "v2":
+            add_import_path("sd2")
+            self.model, H, W = load_sd2_model(self.checkpoint_root(), v2_highres)
+        else:
+            raise ValueError(f"{variant}")
+
+        ae_resolution_f = 8
+
+        self._device = self.model._device
+
+        self.prompt = prompt
+        self.scale = scale
+        self.precision = precision
+        self.precision_scope = autocast if self.precision == "autocast" else nullcontext
+        self._data_shape = (4, H // ae_resolution_f, W // ae_resolution_f)
+
+        self.cond_func = self.model.get_learned_conditioning
+        self.M = 1000
+        noise_schedule = "linear"
+        self.noise_schedule = noise_schedule
+        self.us = self.linear_us(self.M)
+
+    def data_shape(self):
+        return self._data_shape
+
+    @property
+    def σ_max(self):
+        return self.us[0]
+
+    @property
+    def σ_min(self):
+        return self.us[-1]
+
+    @torch.no_grad()
+    def denoise(self, xs, σ, **model_kwargs):
+        with self.precision_scope("cuda"):
+            with self.model.ema_scope():
+                N = xs.shape[0]
+                c = model_kwargs.pop('c')
+                uc = model_kwargs.pop('uc')
+                cond_t, σ = self.time_cond_vec(N, σ)
+                unscaled_xs = xs
+                xs = xs / _sqrt(1 + σ**2)
+                if uc is None or self.scale == 1.:
+                    output = self.model.apply_model(xs, cond_t, c)
+                else:
+                    x_in = torch.cat([xs] * 2)
+                    t_in = torch.cat([cond_t] * 2)
+                    c_in = torch.cat([uc, c])
+                    e_t_uncond, e_t = self.model.apply_model(x_in, t_in, c_in).chunk(2)
+                    output = e_t_uncond + self.scale * (e_t - e_t_uncond)
+
+                if self.model.parameterization == "v":
+                    output = self.model.predict_eps_from_z_and_v(xs, cond_t, output)
+                else:
+                    output = output
+
+                Ds = unscaled_xs - σ * output
+                return Ds
+
+    def cond_info(self, batch_size):
+        prompts = batch_size * [self.prompt]
+        return self.prompts_emb(prompts)
+
+    @torch.no_grad()
+    def prompts_emb(self, prompts):
+        assert isinstance(prompts, list)
+        batch_size = len(prompts)
+        with self.precision_scope("cuda"):
+            with self.model.ema_scope():
+                cond = {}
+                c = self.cond_func(prompts)
+                cond['c'] = c
+                uc = None
+                if self.scale != 1.0:
+                    uc = self.cond_func(batch_size * [""])
+                cond['uc'] = uc
+                return cond
+
+    def unet_is_cond(self):
+        return True
+
+    def use_cls_guidance(self):
+        return False
+
+    def snap_t_to_nearest_tick(self, t):
+        j = np.abs(t - self.us).argmin()
+        return self.us[j], j
+
+    def time_cond_vec(self, N, σ):
+        if isinstance(σ, float):
+            σ, j = self.snap_t_to_nearest_tick(σ)  # σ might change due to snapping
+            cond_t = (self.M - 1) - j
+            cond_t = torch.tensor([cond_t] * N, device=self.device)
+            return cond_t, σ
+        else:
+            assert isinstance(σ, torch.Tensor)
+            σ = σ.reshape(-1).cpu().numpy()
+            σs = []
+            js = []
+            for elem in σ:
+                _σ, _j = self.snap_t_to_nearest_tick(elem)
+                σs.append(_σ)
+                js.append((self.M - 1) - _j)
+
+            cond_t = torch.tensor(js, device=self.device)
+            σs = torch.tensor(σs, device=self.device, dtype=torch.float32).reshape(-1, 1, 1, 1)
+            return cond_t, σs
+
+    @staticmethod
+    def linear_us(M=1000):
+        assert M == 1000
+        β_start = 0.00085
+        β_end = 0.0120
+        βs = np.linspace(β_start**0.5, β_end**0.5, M, dtype=np.float64)**2
+        αs = np.cumprod(1 - βs)
+        us = np.sqrt((1 - αs) / αs)
+        us = us[::-1]
+        return us
+
+    @torch.no_grad()
+    def encode(self, xs):
+        model = self.model
+        with self.precision_scope("cuda"):
+            with self.model.ema_scope():
+                zs = model.get_first_stage_encoding(
+                    model.encode_first_stage(xs)
+                )
+        return zs
+
+    @torch.no_grad()
+    def decode(self, xs):
+        with self.precision_scope("cuda"):
+            with self.model.ema_scope():
+                xs = self.model.decode_first_stage(xs)
+                return xs
+
+
+def test():
+    sd = StableDiffusion("v2", True, "haha", 10.0, "autocast")
+    print(sd)
+
+
+if __name__ == "__main__":
+    test()

adapt_vesde.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+import torch
+from ml_collections.config_flags import config_flags
+
+from sde.config import get_config
+from sde import ddpm, ncsnv2, ncsnpp  # need to import to trigger its registry
+from sde import utils as mutils
+from sde.ema import ExponentialMovingAverage
+
+from adapt import ScoreAdapter
+
+device = torch.device("cuda")
+
+
+def restore_checkpoint(ckpt_dir, state, device):
+    loaded_state = torch.load(ckpt_dir, map_location=device)
+    # state['optimizer'].load_state_dict(loaded_state['optimizer'])
+    state['model'].load_state_dict(loaded_state['model'], strict=False)
+    state['ema'].load_state_dict(loaded_state['ema'])
+    state['step'] = loaded_state['step']
+    return state
+
+
+def save_checkpoint(ckpt_dir, state):
+    saved_state = {
+        'optimizer': state['optimizer'].state_dict(),
+        'model': state['model'].state_dict(),
+        'ema': state['ema'].state_dict(),
+        'step': state['step']
+    }
+    torch.save(saved_state, ckpt_dir)
+
+
+class VESDE(ScoreAdapter):
+    def __init__(self):
+        config = get_config()
+        config.device = device
+        ckpt_fname = self.checkpoint_root() / "sde" / 'checkpoint_127.pth'
+
+        score_model = mutils.create_model(config)
+        ema = ExponentialMovingAverage(
+            score_model.parameters(), decay=config.model.ema_rate
+        )
+        state = dict(model=score_model, ema=ema, step=0)
+        self._data_shape = (
+            config.data.num_channels, config.data.image_size, config.data.image_size
+        )
+
+        self._σ_min = float(config.model.sigma_min * 2)
+
+        state = restore_checkpoint(ckpt_fname, state, device=config.device)
+        ema.copy_to(score_model.parameters())
+
+        score_model.eval()
+        score_model = score_model.module  # remove DataParallel
+
+        self.model = score_model
+        self._device = device
+
+    def data_shape(self):
+        return self._data_shape
+
+    @property
+    def σ_min(self):
+        return self._σ_min
+
+    @torch.no_grad()
+    def denoise(self, xs, σ):
+        N = xs.shape[0]
+        # see Karras eqn. 212-215 for the 1/2 σ correction
+        cond_t = (0.5 * σ) * torch.ones(N, device=self.device)
+        # note that the forward function the model has been modified; see comments
+        n_hat = self.model(xs, cond_t)
+        Ds = xs + σ * n_hat
+        return Ds
+
+    def unet_is_cond(self):
+        return False
+
+    def use_cls_guidance(self):
+        return False
+
+    def snap_t_to_nearest_tick(self, t):
+        return super().snap_t_to_nearest_tick(t)

app.py

@@ -0,0 +1,223 @@
+import numpy as np
+import time
+from pathlib import Path
+import torch
+import imageio
+
+from my.utils import tqdm
+from my.utils.seed import seed_everything
+
+from run_img_sampling import SD, StableDiffusion
+from misc import torch_samps_to_imgs
+from pose import PoseConfig
+
+from run_nerf import VoxConfig
+from voxnerf.utils import every
+from voxnerf.vis import stitch_vis, bad_vis as nerf_vis
+
+from run_sjc import render_one_view, tsr_stats
+from highres_final_vis import highres_render_one_view
+
+import gradio as gr
+import gc
+import os
+
+device_glb = torch.device("cuda")
+
+def vis_routine(y, depth):
+    pane = nerf_vis(y, depth, final_H=256)
+    im = torch_samps_to_imgs(y)[0]
+    depth = depth.cpu().numpy()
+    return pane, im, depth
+
+css = '''
+    .instruction{position: absolute; top: 0;right: 0;margin-top: 0px !important}
+    .arrow{position: absolute;top: 0;right: -110px;margin-top: -8px !important}
+    #component-4, #component-3, #component-10{min-height: 0}
+    .duplicate-button img{margin: 0}
+'''
+
+with gr.Blocks(css=css) as demo:
+    # title
+    gr.Markdown('# [Score Jacobian Chaining](https://github.com/pals-ttic/sjc): Lifting Pretrained 2D Diffusion Models for 3D Generation')
+
+    gr.HTML(f'''
+                <div class="gr-prose" style="max-width: 80%">
+                <h2>Attention - This Space takes over 30min to run!</h2>
+                <p>If the Queue is too long you can run locally or duplicate the Space and run it on your own profile using a (paid) private T4 GPU for training. As each T4 costs US$0.60/h, it should cost < US$1 to train most models using default settings!&nbsp;&nbsp;<a style='display:inline-block' href='https://huggingface.co/spaces/MirageML/sjc?duplicate=true'><img src='https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14' alt='Duplicate Space'></a></p>
+                </div>
+            ''')
+    # inputs
+    prompt = gr.Textbox(label="Prompt", max_lines=1, value="A high quality photo of a delicious burger")
+    iters = gr.Slider(label="Iters", minimum=100, maximum=20000, value=10000, step=100)
+    seed = gr.Slider(label="Seed", minimum=0, maximum=2147483647, step=1, randomize=True)
+    button = gr.Button('Generate')
+
+    # outputs
+    image = gr.Image(label="image", visible=True)
+    # depth = gr.Image(label="depth", visible=True)
+    video = gr.Video(label="video", visible=False)
+    logs = gr.Textbox(label="logging")
+
+    def submit(prompt, iters, seed):
+        start_t = time.time()
+        seed_everything(seed)
+        # cfgs = {'gddpm': {'model': 'm_lsun_256', 'lsun_cat': 'bedroom', 'imgnet_cat': -1}, 'sd': {'variant': 'v1', 'v2_highres': False, 'prompt': 'A high quality photo of a delicious burger', 'scale': 100.0, 'precision': 'autocast'}, 'lr': 0.05, 'n_steps': 10000, 'emptiness_scale': 10, 'emptiness_weight': 10000, 'emptiness_step': 0.5, 'emptiness_multiplier': 20.0, 'depth_weight': 0, 'var_red': True}
+        pose = PoseConfig(rend_hw=64, FoV=60.0, R=1.5)
+        poser = pose.make()
+        sd_model = SD(variant='v1', v2_highres=False, prompt=prompt, scale=100.0, precision='autocast')
+        model = sd_model.make()
+        vox  = VoxConfig(
+                model_type="V_SD", grid_size=100, density_shift=-1.0, c=4,
+                blend_bg_texture=True, bg_texture_hw=4,
+                bbox_len=1.0)
+        vox = vox.make()
+
+        lr = 0.05
+        n_steps = iters
+        emptiness_scale = 10
+        emptiness_weight = 10000
+        emptiness_step = 0.5
+        emptiness_multiplier = 20.0
+        depth_weight = 0
+        var_red = True
+
+        assert model.samps_centered()
+        _, target_H, target_W = model.data_shape()
+        bs = 1
+        aabb = vox.aabb.T.cpu().numpy()
+        vox = vox.to(device_glb)
+        opt = torch.optim.Adamax(vox.opt_params(), lr=lr)
+
+        H, W = poser.H, poser.W
+        Ks, poses, prompt_prefixes = poser.sample_train(n_steps)
+
+        ts = model.us[30:-10]
+
+        same_noise = torch.randn(1, 4, H, W, device=model.device).repeat(bs, 1, 1, 1)
+
+        with tqdm(total=n_steps) as pbar:
+            for i in range(n_steps):
+
+                p = f"{prompt_prefixes[i]} {model.prompt}"
+                score_conds = model.prompts_emb([p])
+
+                y, depth, ws = render_one_view(vox, aabb, H, W, Ks[i], poses[i], return_w=True)
+
+                if isinstance(model, StableDiffusion):
+                    pass
+                else:
+                    y = torch.nn.functional.interpolate(y, (target_H, target_W), mode='bilinear')
+
+                opt.zero_grad()
+
+                with torch.no_grad():
+                    chosen_σs = np.random.choice(ts, bs, replace=False)
+                    chosen_σs = chosen_σs.reshape(-1, 1, 1, 1)
+                    chosen_σs = torch.as_tensor(chosen_σs, device=model.device, dtype=torch.float32)
+                    # chosen_σs = us[i]
+
+                    noise = torch.randn(bs, *y.shape[1:], device=model.device)
+
+                    zs = y + chosen_σs * noise
+                    Ds = model.denoise(zs, chosen_σs, **score_conds)
+
+                    if var_red:
+                        grad = (Ds - y) / chosen_σs
+                    else:
+                        grad = (Ds - zs) / chosen_σs
+
+                    grad = grad.mean(0, keepdim=True)
+
+                y.backward(-grad, retain_graph=True)
+
+                if depth_weight > 0:
+                    center_depth = depth[7:-7, 7:-7]
+                    border_depth_mean = (depth.sum() - center_depth.sum()) / (64*64-50*50)
+                    center_depth_mean = center_depth.mean()
+                    depth_diff = center_depth_mean - border_depth_mean
+                    depth_loss = - torch.log(depth_diff + 1e-12)
+                    depth_loss = depth_weight * depth_loss
+                    depth_loss.backward(retain_graph=True)
+
+                emptiness_loss = torch.log(1 + emptiness_scale * ws).mean()
+                emptiness_loss = emptiness_weight * emptiness_loss
+                if emptiness_step * n_steps <= i:
+                    emptiness_loss *= emptiness_multiplier
+                emptiness_loss.backward()
+
+                opt.step()
+
+
+                # metric.put_scalars()
+
+                with torch.no_grad():
+                    if isinstance(model, StableDiffusion):
+                        y = model.decode(y)
+                    pane, img, depth = vis_routine(y, depth)
+                    yield {
+                        image: gr.update(value=img, visible=True),
+                        video: gr.update(visible=False),
+                        logs: f"Steps: {i}/{n_steps}: \n" + str(tsr_stats(y)),
+                    }
+
+                # TODO: Output pane, img and depth to Gradio
+
+                pbar.update()
+                pbar.set_description(p)
+
+            # TODO: Save Checkpoint
+            with torch.no_grad():
+                n_frames=200
+                factor=4
+                ckpt = vox.state_dict()
+                H, W = poser.H, poser.W
+                vox.eval()
+                K, poses = poser.sample_test(n_frames)
+                del n_frames
+                poses = poses[60:]  # skip the full overhead view; not interesting
+
+                aabb = vox.aabb.T.cpu().numpy()
+                vox = vox.to(device_glb)
+
+                num_imgs = len(poses)
+                all_images = []
+
+                for i in (pbar := tqdm(range(num_imgs))):
+
+                    pose = poses[i]
+                    y, depth = highres_render_one_view(vox, aabb, H, W, K, pose, f=factor)
+                    if isinstance(model, StableDiffusion):
+                        y = model.decode(y)
+                    pane, img, depth = vis_routine(y, depth)
+
+                    # Save img to output
+                    all_images.append(img)
+
+                    yield {
+                        image: gr.update(value=img, visible=True),
+                        video: gr.update(visible=False),
+                        logs: str(tsr_stats(y)),
+                    }
+
+                output_video = "/tmp/tmp.mp4"
+
+                imageio.mimwrite(output_video, all_images, quality=8, fps=10)
+
+                end_t = time.time()
+
+                yield {
+                    image: gr.update(value=img, visible=False),
+                    video: gr.update(value=output_video, visible=True),
+                    logs: f"Generation Finished in {(end_t - start_t)/ 60:.4f} minutes!",
+                }
+
+    button.click(
+            submit,
+            [prompt, iters, seed],
+            [image, video, logs]
+    )
+
+# concurrency_count: only allow ONE running progress, else GPU will OOM.
+demo.queue(concurrency_count=1)
+demo.launch()

env.json

@@ -0,0 +1,3 @@
+{
+    "data_root": "release"
+}

guided_diffusion/README.md

@@ -0,0 +1,5 @@
+Selected modules from OpenAI's [guided diffusion](https://github.com/openai/guided-diffusion), retrieved at commit `22e0df8183507e13a7813f8d38d51b072ca1e67c`
+
+It's a bare minimum set of files needed to run their pretrained models. You can download these model checkpoints following the instructions in their repository README
+
+Some modifications are made to remove the distributed processing utilities in order to reduce code complexity.

guided_diffusion/__init__.py

@@ -0,0 +1,3 @@
+"""
+Codebase for "Improved Denoising Diffusion Probabilistic Models".
+"""

guided_diffusion/fp16_util.py

@@ -0,0 +1,237 @@
+"""
+Helpers to train with 16-bit precision.
+"""
+
+import numpy as np
+import torch as th
+import torch.nn as nn
+from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
+
+# from . import logger
+
+INITIAL_LOG_LOSS_SCALE = 20.0
+
+
+def convert_module_to_f16(l):
+    """
+    Convert primitive modules to float16.
+    """
+    if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Conv3d)):
+        l.weight.data = l.weight.data.half()
+        if l.bias is not None:
+            l.bias.data = l.bias.data.half()
+
+
+def convert_module_to_f32(l):
+    """
+    Convert primitive modules to float32, undoing convert_module_to_f16().
+    """
+    if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Conv3d)):
+        l.weight.data = l.weight.data.float()
+        if l.bias is not None:
+            l.bias.data = l.bias.data.float()
+
+
+def make_master_params(param_groups_and_shapes):
+    """
+    Copy model parameters into a (differently-shaped) list of full-precision
+    parameters.
+    """
+    master_params = []
+    for param_group, shape in param_groups_and_shapes:
+        master_param = nn.Parameter(
+            _flatten_dense_tensors(
+                [param.detach().float() for (_, param) in param_group]
+            ).view(shape)
+        )
+        master_param.requires_grad = True
+        master_params.append(master_param)
+    return master_params
+
+
+def model_grads_to_master_grads(param_groups_and_shapes, master_params):
+    """
+    Copy the gradients from the model parameters into the master parameters
+    from make_master_params().
+    """
+    for master_param, (param_group, shape) in zip(
+        master_params, param_groups_and_shapes
+    ):
+        master_param.grad = _flatten_dense_tensors(
+            [param_grad_or_zeros(param) for (_, param) in param_group]
+        ).view(shape)
+
+
+def master_params_to_model_params(param_groups_and_shapes, master_params):
+    """
+    Copy the master parameter data back into the model parameters.
+    """
+    # Without copying to a list, if a generator is passed, this will
+    # silently not copy any parameters.
+    for master_param, (param_group, _) in zip(master_params, param_groups_and_shapes):
+        for (_, param), unflat_master_param in zip(
+            param_group, unflatten_master_params(param_group, master_param.view(-1))
+        ):
+            param.detach().copy_(unflat_master_param)
+
+
+def unflatten_master_params(param_group, master_param):
+    return _unflatten_dense_tensors(master_param, [param for (_, param) in param_group])
+
+
+def get_param_groups_and_shapes(named_model_params):
+    named_model_params = list(named_model_params)
+    scalar_vector_named_params = (
+        [(n, p) for (n, p) in named_model_params if p.ndim <= 1],
+        (-1),
+    )
+    matrix_named_params = (
+        [(n, p) for (n, p) in named_model_params if p.ndim > 1],
+        (1, -1),
+    )
+    return [scalar_vector_named_params, matrix_named_params]
+
+
+def master_params_to_state_dict(
+    model, param_groups_and_shapes, master_params, use_fp16
+):
+    if use_fp16:
+        state_dict = model.state_dict()
+        for master_param, (param_group, _) in zip(
+            master_params, param_groups_and_shapes
+        ):
+            for (name, _), unflat_master_param in zip(
+                param_group, unflatten_master_params(param_group, master_param.view(-1))
+            ):
+                assert name in state_dict
+                state_dict[name] = unflat_master_param
+    else:
+        state_dict = model.state_dict()
+        for i, (name, _value) in enumerate(model.named_parameters()):
+            assert name in state_dict
+            state_dict[name] = master_params[i]
+    return state_dict
+
+
+def state_dict_to_master_params(model, state_dict, use_fp16):
+    if use_fp16:
+        named_model_params = [
+            (name, state_dict[name]) for name, _ in model.named_parameters()
+        ]
+        param_groups_and_shapes = get_param_groups_and_shapes(named_model_params)
+        master_params = make_master_params(param_groups_and_shapes)
+    else:
+        master_params = [state_dict[name] for name, _ in model.named_parameters()]
+    return master_params
+
+
+def zero_master_grads(master_params):
+    for param in master_params:
+        param.grad = None
+
+
+def zero_grad(model_params):
+    for param in model_params:
+        # Taken from https://pytorch.org/docs/stable/_modules/torch/optim/optimizer.html#Optimizer.add_param_group
+        if param.grad is not None:
+            param.grad.detach_()
+            param.grad.zero_()
+
+
+def param_grad_or_zeros(param):
+    if param.grad is not None:
+        return param.grad.data.detach()
+    else:
+        return th.zeros_like(param)
+
+
+class MixedPrecisionTrainer:
+    def __init__(
+        self,
+        *,
+        model,
+        use_fp16=False,
+        fp16_scale_growth=1e-3,
+        initial_lg_loss_scale=INITIAL_LOG_LOSS_SCALE,
+    ):
+        self.model = model
+        self.use_fp16 = use_fp16
+        self.fp16_scale_growth = fp16_scale_growth
+
+        self.model_params = list(self.model.parameters())
+        self.master_params = self.model_params
+        self.param_groups_and_shapes = None
+        self.lg_loss_scale = initial_lg_loss_scale
+
+        if self.use_fp16:
+            self.param_groups_and_shapes = get_param_groups_and_shapes(
+                self.model.named_parameters()
+            )
+            self.master_params = make_master_params(self.param_groups_and_shapes)
+            self.model.convert_to_fp16()
+
+    def zero_grad(self):
+        zero_grad(self.model_params)
+
+    def backward(self, loss: th.Tensor):
+        if self.use_fp16:
+            loss_scale = 2 ** self.lg_loss_scale
+            (loss * loss_scale).backward()
+        else:
+            loss.backward()
+
+    def optimize(self, opt: th.optim.Optimizer):
+        if self.use_fp16:
+            return self._optimize_fp16(opt)
+        else:
+            return self._optimize_normal(opt)
+
+    def _optimize_fp16(self, opt: th.optim.Optimizer):
+        logger.logkv_mean("lg_loss_scale", self.lg_loss_scale)
+        model_grads_to_master_grads(self.param_groups_and_shapes, self.master_params)
+        grad_norm, param_norm = self._compute_norms(grad_scale=2 ** self.lg_loss_scale)
+        if check_overflow(grad_norm):
+            self.lg_loss_scale -= 1
+            logger.log(f"Found NaN, decreased lg_loss_scale to {self.lg_loss_scale}")
+            zero_master_grads(self.master_params)
+            return False
+
+        logger.logkv_mean("grad_norm", grad_norm)
+        logger.logkv_mean("param_norm", param_norm)
+
+        for p in self.master_params:
+            p.grad.mul_(1.0 / (2 ** self.lg_loss_scale))
+        opt.step()
+        zero_master_grads(self.master_params)
+        master_params_to_model_params(self.param_groups_and_shapes, self.master_params)
+        self.lg_loss_scale += self.fp16_scale_growth
+        return True
+
+    def _optimize_normal(self, opt: th.optim.Optimizer):
+        grad_norm, param_norm = self._compute_norms()
+        logger.logkv_mean("grad_norm", grad_norm)
+        logger.logkv_mean("param_norm", param_norm)
+        opt.step()
+        return True
+
+    def _compute_norms(self, grad_scale=1.0):
+        grad_norm = 0.0
+        param_norm = 0.0
+        for p in self.master_params:
+            with th.no_grad():
+                param_norm += th.norm(p, p=2, dtype=th.float32).item() ** 2
+                if p.grad is not None:
+                    grad_norm += th.norm(p.grad, p=2, dtype=th.float32).item() ** 2
+        return np.sqrt(grad_norm) / grad_scale, np.sqrt(param_norm)
+
+    def master_params_to_state_dict(self, master_params):
+        return master_params_to_state_dict(
+            self.model, self.param_groups_and_shapes, master_params, self.use_fp16
+        )
+
+    def state_dict_to_master_params(self, state_dict):
+        return state_dict_to_master_params(self.model, state_dict, self.use_fp16)
+
+
+def check_overflow(value):
+    return (value == float("inf")) or (value == -float("inf")) or (value != value)

guided_diffusion/gaussian_diffusion.py

@@ -0,0 +1,908 @@
+"""
+This code started out as a PyTorch port of Ho et al's diffusion models:
+https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/diffusion_utils_2.py
+
+Docstrings have been added, as well as DDIM sampling and a new collection of beta schedules.
+"""
+
+import enum
+import math
+
+import numpy as np
+import torch as th
+
+from .nn import mean_flat
+from .losses import normal_kl, discretized_gaussian_log_likelihood
+
+
+def get_named_beta_schedule(schedule_name, num_diffusion_timesteps):
+    """
+    Get a pre-defined beta schedule for the given name.
+
+    The beta schedule library consists of beta schedules which remain similar
+    in the limit of num_diffusion_timesteps.
+    Beta schedules may be added, but should not be removed or changed once
+    they are committed to maintain backwards compatibility.
+    """
+    if schedule_name == "linear":
+        # Linear schedule from Ho et al, extended to work for any number of
+        # diffusion steps.
+        scale = 1000 / num_diffusion_timesteps
+        beta_start = scale * 0.0001
+        beta_end = scale * 0.02
+        return np.linspace(
+            beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64
+        )
+    elif schedule_name == "cosine":
+        return betas_for_alpha_bar(
+            num_diffusion_timesteps,
+            lambda t: math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2,
+        )
+    else:
+        raise NotImplementedError(f"unknown beta schedule: {schedule_name}")
+
+
+def betas_for_alpha_bar(num_diffusion_timesteps, alpha_bar, max_beta=0.999):
+    """
+    Create a beta schedule that discretizes the given alpha_t_bar function,
+    which defines the cumulative product of (1-beta) over time from t = [0,1].
+
+    :param num_diffusion_timesteps: the number of betas to produce.
+    :param alpha_bar: a lambda that takes an argument t from 0 to 1 and
+                      produces the cumulative product of (1-beta) up to that
+                      part of the diffusion process.
+    :param max_beta: the maximum beta to use; use values lower than 1 to
+                     prevent singularities.
+    """
+    betas = []
+    for i in range(num_diffusion_timesteps):
+        t1 = i / num_diffusion_timesteps
+        t2 = (i + 1) / num_diffusion_timesteps
+        betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta))
+    return np.array(betas)
+
+
+class ModelMeanType(enum.Enum):
+    """
+    Which type of output the model predicts.
+    """
+
+    PREVIOUS_X = enum.auto()  # the model predicts x_{t-1}
+    START_X = enum.auto()  # the model predicts x_0
+    EPSILON = enum.auto()  # the model predicts epsilon
+
+
+class ModelVarType(enum.Enum):
+    """
+    What is used as the model's output variance.
+
+    The LEARNED_RANGE option has been added to allow the model to predict
+    values between FIXED_SMALL and FIXED_LARGE, making its job easier.
+    """
+
+    LEARNED = enum.auto()
+    FIXED_SMALL = enum.auto()
+    FIXED_LARGE = enum.auto()
+    LEARNED_RANGE = enum.auto()
+
+
+class LossType(enum.Enum):
+    MSE = enum.auto()  # use raw MSE loss (and KL when learning variances)
+    RESCALED_MSE = (
+        enum.auto()
+    )  # use raw MSE loss (with RESCALED_KL when learning variances)
+    KL = enum.auto()  # use the variational lower-bound
+    RESCALED_KL = enum.auto()  # like KL, but rescale to estimate the full VLB
+
+    def is_vb(self):
+        return self == LossType.KL or self == LossType.RESCALED_KL
+
+
+class GaussianDiffusion:
+    """
+    Utilities for training and sampling diffusion models.
+
+    Ported directly from here, and then adapted over time to further experimentation.
+    https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/diffusion_utils_2.py#L42
+
+    :param betas: a 1-D numpy array of betas for each diffusion timestep,
+                  starting at T and going to 1.
+    :param model_mean_type: a ModelMeanType determining what the model outputs.
+    :param model_var_type: a ModelVarType determining how variance is output.
+    :param loss_type: a LossType determining the loss function to use.
+    :param rescale_timesteps: if True, pass floating point timesteps into the
+                              model so that they are always scaled like in the
+                              original paper (0 to 1000).
+    """
+
+    def __init__(
+        self,
+        *,
+        betas,
+        model_mean_type,
+        model_var_type,
+        loss_type,
+        rescale_timesteps=False,
+    ):
+        self.model_mean_type = model_mean_type
+        self.model_var_type = model_var_type
+        self.loss_type = loss_type
+        self.rescale_timesteps = rescale_timesteps
+
+        # Use float64 for accuracy.
+        betas = np.array(betas, dtype=np.float64)
+        self.betas = betas
+        assert len(betas.shape) == 1, "betas must be 1-D"
+        assert (betas > 0).all() and (betas <= 1).all()
+
+        self.num_timesteps = int(betas.shape[0])
+
+        alphas = 1.0 - betas
+        self.alphas_cumprod = np.cumprod(alphas, axis=0)
+        self.alphas_cumprod_prev = np.append(1.0, self.alphas_cumprod[:-1])
+        self.alphas_cumprod_next = np.append(self.alphas_cumprod[1:], 0.0)
+        assert self.alphas_cumprod_prev.shape == (self.num_timesteps,)
+
+        # calculations for diffusion q(x_t | x_{t-1}) and others
+        self.sqrt_alphas_cumprod = np.sqrt(self.alphas_cumprod)
+        self.sqrt_one_minus_alphas_cumprod = np.sqrt(1.0 - self.alphas_cumprod)
+        self.log_one_minus_alphas_cumprod = np.log(1.0 - self.alphas_cumprod)
+        self.sqrt_recip_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod)
+        self.sqrt_recipm1_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod - 1)
+
+        # calculations for posterior q(x_{t-1} | x_t, x_0)
+        self.posterior_variance = (
+            betas * (1.0 - self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)
+        )
+        # log calculation clipped because the posterior variance is 0 at the
+        # beginning of the diffusion chain.
+        self.posterior_log_variance_clipped = np.log(
+            np.append(self.posterior_variance[1], self.posterior_variance[1:])
+        )
+        self.posterior_mean_coef1 = (
+            betas * np.sqrt(self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)
+        )
+        self.posterior_mean_coef2 = (
+            (1.0 - self.alphas_cumprod_prev)
+            * np.sqrt(alphas)
+            / (1.0 - self.alphas_cumprod)
+        )
+
+    def q_mean_variance(self, x_start, t):
+        """
+        Get the distribution q(x_t | x_0).
+
+        :param x_start: the [N x C x ...] tensor of noiseless inputs.
+        :param t: the number of diffusion steps (minus 1). Here, 0 means one step.
+        :return: A tuple (mean, variance, log_variance), all of x_start's shape.
+        """
+        mean = (
+            _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start
+        )
+        variance = _extract_into_tensor(1.0 - self.alphas_cumprod, t, x_start.shape)
+        log_variance = _extract_into_tensor(
+            self.log_one_minus_alphas_cumprod, t, x_start.shape
+        )
+        return mean, variance, log_variance
+
+    def q_sample(self, x_start, t, noise=None):
+        """
+        Diffuse the data for a given number of diffusion steps.
+
+        In other words, sample from q(x_t | x_0).
+
+        :param x_start: the initial data batch.
+        :param t: the number of diffusion steps (minus 1). Here, 0 means one step.
+        :param noise: if specified, the split-out normal noise.
+        :return: A noisy version of x_start.
+        """
+        if noise is None:
+            noise = th.randn_like(x_start)
+        assert noise.shape == x_start.shape
+        return (
+            _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start
+            + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape)
+            * noise
+        )
+
+    def q_posterior_mean_variance(self, x_start, x_t, t):
+        """
+        Compute the mean and variance of the diffusion posterior:
+
+            q(x_{t-1} | x_t, x_0)
+
+        """
+        assert x_start.shape == x_t.shape
+        posterior_mean = (
+            _extract_into_tensor(self.posterior_mean_coef1, t, x_t.shape) * x_start
+            + _extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) * x_t
+        )
+        posterior_variance = _extract_into_tensor(self.posterior_variance, t, x_t.shape)
+        posterior_log_variance_clipped = _extract_into_tensor(
+            self.posterior_log_variance_clipped, t, x_t.shape
+        )
+        assert (
+            posterior_mean.shape[0]
+            == posterior_variance.shape[0]
+            == posterior_log_variance_clipped.shape[0]
+            == x_start.shape[0]
+        )
+        return posterior_mean, posterior_variance, posterior_log_variance_clipped
+
+    def p_mean_variance(
+        self, model, x, t, clip_denoised=True, denoised_fn=None, model_kwargs=None
+    ):
+        """
+        Apply the model to get p(x_{t-1} | x_t), as well as a prediction of
+        the initial x, x_0.
+
+        :param model: the model, which takes a signal and a batch of timesteps
+                      as input.
+        :param x: the [N x C x ...] tensor at time t.
+        :param t: a 1-D Tensor of timesteps.
+        :param clip_denoised: if True, clip the denoised signal into [-1, 1].
+        :param denoised_fn: if not None, a function which applies to the
+            x_start prediction before it is used to sample. Applies before
+            clip_denoised.
+        :param model_kwargs: if not None, a dict of extra keyword arguments to
+            pass to the model. This can be used for conditioning.
+        :return: a dict with the following keys:
+                 - 'mean': the model mean output.
+                 - 'variance': the model variance output.
+                 - 'log_variance': the log of 'variance'.
+                 - 'pred_xstart': the prediction for x_0.
+        """
+        if model_kwargs is None:
+            model_kwargs = {}
+
+        B, C = x.shape[:2]
+        assert t.shape == (B,)
+        model_output = model(x, self._scale_timesteps(t), **model_kwargs)
+
+        if self.model_var_type in [ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE]:
+            assert model_output.shape == (B, C * 2, *x.shape[2:])
+            model_output, model_var_values = th.split(model_output, C, dim=1)
+            if self.model_var_type == ModelVarType.LEARNED:
+                model_log_variance = model_var_values
+                model_variance = th.exp(model_log_variance)
+            else:
+                min_log = _extract_into_tensor(
+                    self.posterior_log_variance_clipped, t, x.shape
+                )
+                max_log = _extract_into_tensor(np.log(self.betas), t, x.shape)
+                # The model_var_values is [-1, 1] for [min_var, max_var].
+                frac = (model_var_values + 1) / 2
+                model_log_variance = frac * max_log + (1 - frac) * min_log
+                model_variance = th.exp(model_log_variance)
+        else:
+            model_variance, model_log_variance = {
+                # for fixedlarge, we set the initial (log-)variance like so
+                # to get a better decoder log likelihood.
+                ModelVarType.FIXED_LARGE: (
+                    np.append(self.posterior_variance[1], self.betas[1:]),
+                    np.log(np.append(self.posterior_variance[1], self.betas[1:])),
+                ),
+                ModelVarType.FIXED_SMALL: (
+                    self.posterior_variance,
+                    self.posterior_log_variance_clipped,
+                ),
+            }[self.model_var_type]
+            model_variance = _extract_into_tensor(model_variance, t, x.shape)
+            model_log_variance = _extract_into_tensor(model_log_variance, t, x.shape)
+
+        def process_xstart(x):
+            if denoised_fn is not None:
+                x = denoised_fn(x)
+            if clip_denoised:
+                return x.clamp(-1, 1)
+            return x
+
+        if self.model_mean_type == ModelMeanType.PREVIOUS_X:
+            pred_xstart = process_xstart(
+                self._predict_xstart_from_xprev(x_t=x, t=t, xprev=model_output)
+            )
+            model_mean = model_output
+        elif self.model_mean_type in [ModelMeanType.START_X, ModelMeanType.EPSILON]:
+            if self.model_mean_type == ModelMeanType.START_X:
+                pred_xstart = process_xstart(model_output)
+            else:
+                pred_xstart = process_xstart(
+                    self._predict_xstart_from_eps(x_t=x, t=t, eps=model_output)
+                )
+            model_mean, _, _ = self.q_posterior_mean_variance(
+                x_start=pred_xstart, x_t=x, t=t
+            )
+        else:
+            raise NotImplementedError(self.model_mean_type)
+
+        assert (
+            model_mean.shape == model_log_variance.shape == pred_xstart.shape == x.shape
+        )
+        return {
+            "mean": model_mean,
+            "variance": model_variance,
+            "log_variance": model_log_variance,
+            "pred_xstart": pred_xstart,
+        }
+
+    def _predict_xstart_from_eps(self, x_t, t, eps):
+        assert x_t.shape == eps.shape
+        return (
+            _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t
+            - _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * eps
+        )
+
+    def _predict_xstart_from_xprev(self, x_t, t, xprev):
+        assert x_t.shape == xprev.shape
+        return (  # (xprev - coef2*x_t) / coef1
+            _extract_into_tensor(1.0 / self.posterior_mean_coef1, t, x_t.shape) * xprev
+            - _extract_into_tensor(
+                self.posterior_mean_coef2 / self.posterior_mean_coef1, t, x_t.shape
+            )
+            * x_t
+        )
+
+    def _predict_eps_from_xstart(self, x_t, t, pred_xstart):
+        return (
+            _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t
+            - pred_xstart
+        ) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)
+
+    def _scale_timesteps(self, t):
+        if self.rescale_timesteps:
+            return t.float() * (1000.0 / self.num_timesteps)
+        return t
+
+    def condition_mean(self, cond_fn, p_mean_var, x, t, model_kwargs=None):
+        """
+        Compute the mean for the previous step, given a function cond_fn that
+        computes the gradient of a conditional log probability with respect to
+        x. In particular, cond_fn computes grad(log(p(y|x))), and we want to
+        condition on y.
+
+        This uses the conditioning strategy from Sohl-Dickstein et al. (2015).
+        """
+        gradient = cond_fn(x, self._scale_timesteps(t), **model_kwargs)
+        new_mean = (
+            p_mean_var["mean"].float() + p_mean_var["variance"] * gradient.float()
+        )
+        return new_mean
+
+    def condition_score(self, cond_fn, p_mean_var, x, t, model_kwargs=None):
+        """
+        Compute what the p_mean_variance output would have been, should the
+        model's score function be conditioned by cond_fn.
+
+        See condition_mean() for details on cond_fn.
+
+        Unlike condition_mean(), this instead uses the conditioning strategy
+        from Song et al (2020).
+        """
+        alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape)
+
+        eps = self._predict_eps_from_xstart(x, t, p_mean_var["pred_xstart"])
+        eps = eps - (1 - alpha_bar).sqrt() * cond_fn(
+            x, self._scale_timesteps(t), **model_kwargs
+        )
+
+        out = p_mean_var.copy()
+        out["pred_xstart"] = self._predict_xstart_from_eps(x, t, eps)
+        out["mean"], _, _ = self.q_posterior_mean_variance(
+            x_start=out["pred_xstart"], x_t=x, t=t
+        )
+        return out
+
+    def p_sample(
+        self,
+        model,
+        x,
+        t,
+        clip_denoised=True,
+        denoised_fn=None,
+        cond_fn=None,
+        model_kwargs=None,
+    ):
+        """
+        Sample x_{t-1} from the model at the given timestep.
+
+        :param model: the model to sample from.
+        :param x: the current tensor at x_{t-1}.
+        :param t: the value of t, starting at 0 for the first diffusion step.
+        :param clip_denoised: if True, clip the x_start prediction to [-1, 1].
+        :param denoised_fn: if not None, a function which applies to the
+            x_start prediction before it is used to sample.
+        :param cond_fn: if not None, this is a gradient function that acts
+                        similarly to the model.
+        :param model_kwargs: if not None, a dict of extra keyword arguments to
+            pass to the model. This can be used for conditioning.
+        :return: a dict containing the following keys:
+                 - 'sample': a random sample from the model.
+                 - 'pred_xstart': a prediction of x_0.
+        """
+        out = self.p_mean_variance(
+            model,
+            x,
+            t,
+            clip_denoised=clip_denoised,
+            denoised_fn=denoised_fn,
+            model_kwargs=model_kwargs,
+        )
+        noise = th.randn_like(x)
+        nonzero_mask = (
+            (t != 0).float().view(-1, *([1] * (len(x.shape) - 1)))
+        )  # no noise when t == 0
+        if cond_fn is not None:
+            out["mean"] = self.condition_mean(
+                cond_fn, out, x, t, model_kwargs=model_kwargs
+            )
+        sample = out["mean"] + nonzero_mask * th.exp(0.5 * out["log_variance"]) * noise
+        return {"sample": sample, "pred_xstart": out["pred_xstart"]}
+
+    def p_sample_loop(
+        self,
+        model,
+        shape,
+        noise=None,
+        clip_denoised=True,
+        denoised_fn=None,
+        cond_fn=None,
+        model_kwargs=None,
+        device=None,
+        progress=False,
+    ):
+        """
+        Generate samples from the model.
+
+        :param model: the model module.
+        :param shape: the shape of the samples, (N, C, H, W).
+        :param noise: if specified, the noise from the encoder to sample.
+                      Should be of the same shape as `shape`.
+        :param clip_denoised: if True, clip x_start predictions to [-1, 1].
+        :param denoised_fn: if not None, a function which applies to the
+            x_start prediction before it is used to sample.
+        :param cond_fn: if not None, this is a gradient function that acts
+                        similarly to the model.
+        :param model_kwargs: if not None, a dict of extra keyword arguments to
+            pass to the model. This can be used for conditioning.
+        :param device: if specified, the device to create the samples on.
+                       If not specified, use a model parameter's device.
+        :param progress: if True, show a tqdm progress bar.
+        :return: a non-differentiable batch of samples.
+        """
+        final = None
+        for sample in self.p_sample_loop_progressive(
+            model,
+            shape,
+            noise=noise,
+            clip_denoised=clip_denoised,
+            denoised_fn=denoised_fn,
+            cond_fn=cond_fn,
+            model_kwargs=model_kwargs,
+            device=device,
+            progress=progress,
+        ):
+            final = sample
+        return final["sample"]
+
+    def p_sample_loop_progressive(
+        self,
+        model,
+        shape,
+        noise=None,
+        clip_denoised=True,
+        denoised_fn=None,
+        cond_fn=None,
+        model_kwargs=None,
+        device=None,
+        progress=False,
+    ):
+        """
+        Generate samples from the model and yield intermediate samples from
+        each timestep of diffusion.
+
+        Arguments are the same as p_sample_loop().
+        Returns a generator over dicts, where each dict is the return value of
+        p_sample().
+        """
+        if device is None:
+            device = next(model.parameters()).device
+        assert isinstance(shape, (tuple, list))
+        if noise is not None:
+            img = noise
+        else:
+            img = th.randn(*shape, device=device)
+        indices = list(range(self.num_timesteps))[::-1]
+
+        if progress:
+            # Lazy import so that we don't depend on tqdm.
+            from tqdm.auto import tqdm
+
+            indices = tqdm(indices)
+
+        for i in indices:
+            t = th.tensor([i] * shape[0], device=device)
+            with th.no_grad():
+                out = self.p_sample(
+                    model,
+                    img,
+                    t,
+                    clip_denoised=clip_denoised,
+                    denoised_fn=denoised_fn,
+                    cond_fn=cond_fn,
+                    model_kwargs=model_kwargs,
+                )
+                yield out
+                img = out["sample"]
+
+    def ddim_sample(
+        self,
+        model,
+        x,
+        t,
+        clip_denoised=True,
+        denoised_fn=None,
+        cond_fn=None,
+        model_kwargs=None,
+        eta=0.0,
+    ):
+        """
+        Sample x_{t-1} from the model using DDIM.
+
+        Same usage as p_sample().
+        """
+        out = self.p_mean_variance(
+            model,
+            x,
+            t,
+            clip_denoised=clip_denoised,
+            denoised_fn=denoised_fn,
+            model_kwargs=model_kwargs,
+        )
+        if cond_fn is not None:
+            out = self.condition_score(cond_fn, out, x, t, model_kwargs=model_kwargs)
+
+        # Usually our model outputs epsilon, but we re-derive it
+        # in case we used x_start or x_prev prediction.
+        eps = self._predict_eps_from_xstart(x, t, out["pred_xstart"])
+
+        alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape)
+        alpha_bar_prev = _extract_into_tensor(self.alphas_cumprod_prev, t, x.shape)
+        sigma = (
+            eta
+            * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar))
+            * th.sqrt(1 - alpha_bar / alpha_bar_prev)
+        )
+        # Equation 12.
+        noise = th.randn_like(x)
+        mean_pred = (
+            out["pred_xstart"] * th.sqrt(alpha_bar_prev)
+            + th.sqrt(1 - alpha_bar_prev - sigma ** 2) * eps
+        )
+        nonzero_mask = (
+            (t != 0).float().view(-1, *([1] * (len(x.shape) - 1)))
+        )  # no noise when t == 0
+        sample = mean_pred + nonzero_mask * sigma * noise
+        return {"sample": sample, "pred_xstart": out["pred_xstart"]}
+
+    def ddim_reverse_sample(
+        self,
+        model,
+        x,
+        t,
+        clip_denoised=True,
+        denoised_fn=None,
+        model_kwargs=None,
+        eta=0.0,
+    ):
+        """
+        Sample x_{t+1} from the model using DDIM reverse ODE.
+        """
+        assert eta == 0.0, "Reverse ODE only for deterministic path"
+        out = self.p_mean_variance(
+            model,
+            x,
+            t,
+            clip_denoised=clip_denoised,
+            denoised_fn=denoised_fn,
+            model_kwargs=model_kwargs,
+        )
+        # Usually our model outputs epsilon, but we re-derive it
+        # in case we used x_start or x_prev prediction.
+        eps = (
+            _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x.shape) * x
+            - out["pred_xstart"]
+        ) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x.shape)
+        alpha_bar_next = _extract_into_tensor(self.alphas_cumprod_next, t, x.shape)
+
+        # Equation 12. reversed
+        mean_pred = (
+            out["pred_xstart"] * th.sqrt(alpha_bar_next)
+            + th.sqrt(1 - alpha_bar_next) * eps
+        )
+
+        return {"sample": mean_pred, "pred_xstart": out["pred_xstart"]}
+
+    def ddim_sample_loop(
+        self,
+        model,
+        shape,
+        noise=None,
+        clip_denoised=True,
+        denoised_fn=None,
+        cond_fn=None,
+        model_kwargs=None,
+        device=None,
+        progress=False,
+        eta=0.0,
+    ):
+        """
+        Generate samples from the model using DDIM.
+
+        Same usage as p_sample_loop().
+        """
+        final = None
+        for sample in self.ddim_sample_loop_progressive(
+            model,
+            shape,
+            noise=noise,
+            clip_denoised=clip_denoised,
+            denoised_fn=denoised_fn,
+            cond_fn=cond_fn,
+            model_kwargs=model_kwargs,
+            device=device,
+            progress=progress,
+            eta=eta,
+        ):
+            final = sample
+        return final["sample"]
+
+    def ddim_sample_loop_progressive(
+        self,
+        model,
+        shape,
+        noise=None,
+        clip_denoised=True,
+        denoised_fn=None,
+        cond_fn=None,
+        model_kwargs=None,
+        device=None,
+        progress=False,
+        eta=0.0,
+    ):
+        """
+        Use DDIM to sample from the model and yield intermediate samples from
+        each timestep of DDIM.
+
+        Same usage as p_sample_loop_progressive().
+        """
+        if device is None:
+            device = next(model.parameters()).device
+        assert isinstance(shape, (tuple, list))
+        if noise is not None:
+            img = noise
+        else:
+            img = th.randn(*shape, device=device)
+        indices = list(range(self.num_timesteps))[::-1]
+
+        if progress:
+            # Lazy import so that we don't depend on tqdm.
+            from tqdm.auto import tqdm
+
+            indices = tqdm(indices)
+
+        for i in indices:
+            t = th.tensor([i] * shape[0], device=device)
+            with th.no_grad():
+                out = self.ddim_sample(
+                    model,
+                    img,
+                    t,
+                    clip_denoised=clip_denoised,
+                    denoised_fn=denoised_fn,
+                    cond_fn=cond_fn,
+                    model_kwargs=model_kwargs,
+                    eta=eta,
+                )
+                yield out
+                img = out["sample"]
+
+    def _vb_terms_bpd(
+        self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None
+    ):
+        """
+        Get a term for the variational lower-bound.
+
+        The resulting units are bits (rather than nats, as one might expect).
+        This allows for comparison to other papers.
+
+        :return: a dict with the following keys:
+                 - 'output': a shape [N] tensor of NLLs or KLs.
+                 - 'pred_xstart': the x_0 predictions.
+        """
+        true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance(
+            x_start=x_start, x_t=x_t, t=t
+        )
+        out = self.p_mean_variance(
+            model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs
+        )
+        kl = normal_kl(
+            true_mean, true_log_variance_clipped, out["mean"], out["log_variance"]
+        )
+        kl = mean_flat(kl) / np.log(2.0)
+
+        decoder_nll = -discretized_gaussian_log_likelihood(
+            x_start, means=out["mean"], log_scales=0.5 * out["log_variance"]
+        )
+        assert decoder_nll.shape == x_start.shape
+        decoder_nll = mean_flat(decoder_nll) / np.log(2.0)
+
+        # At the first timestep return the decoder NLL,
+        # otherwise return KL(q(x_{t-1}|x_t,x_0) || p(x_{t-1}|x_t))
+        output = th.where((t == 0), decoder_nll, kl)
+        return {"output": output, "pred_xstart": out["pred_xstart"]}
+
+    def training_losses(self, model, x_start, t, model_kwargs=None, noise=None):
+        """
+        Compute training losses for a single timestep.
+
+        :param model: the model to evaluate loss on.
+        :param x_start: the [N x C x ...] tensor of inputs.
+        :param t: a batch of timestep indices.
+        :param model_kwargs: if not None, a dict of extra keyword arguments to
+            pass to the model. This can be used for conditioning.
+        :param noise: if specified, the specific Gaussian noise to try to remove.
+        :return: a dict with the key "loss" containing a tensor of shape [N].
+                 Some mean or variance settings may also have other keys.
+        """
+        if model_kwargs is None:
+            model_kwargs = {}
+        if noise is None:
+            noise = th.randn_like(x_start)
+        x_t = self.q_sample(x_start, t, noise=noise)
+
+        terms = {}
+
+        if self.loss_type == LossType.KL or self.loss_type == LossType.RESCALED_KL:
+            terms["loss"] = self._vb_terms_bpd(
+                model=model,
+                x_start=x_start,
+                x_t=x_t,
+                t=t,
+                clip_denoised=False,
+                model_kwargs=model_kwargs,
+            )["output"]
+            if self.loss_type == LossType.RESCALED_KL:
+                terms["loss"] *= self.num_timesteps
+        elif self.loss_type == LossType.MSE or self.loss_type == LossType.RESCALED_MSE:
+            model_output = model(x_t, self._scale_timesteps(t), **model_kwargs)
+
+            if self.model_var_type in [
+                ModelVarType.LEARNED,
+                ModelVarType.LEARNED_RANGE,
+            ]:
+                B, C = x_t.shape[:2]
+                assert model_output.shape == (B, C * 2, *x_t.shape[2:])
+                model_output, model_var_values = th.split(model_output, C, dim=1)
+                # Learn the variance using the variational bound, but don't let
+                # it affect our mean prediction.
+                frozen_out = th.cat([model_output.detach(), model_var_values], dim=1)
+                terms["vb"] = self._vb_terms_bpd(
+                    model=lambda *args, r=frozen_out: r,
+                    x_start=x_start,
+                    x_t=x_t,
+                    t=t,
+                    clip_denoised=False,
+                )["output"]
+                if self.loss_type == LossType.RESCALED_MSE:
+                    # Divide by 1000 for equivalence with initial implementation.
+                    # Without a factor of 1/1000, the VB term hurts the MSE term.
+                    terms["vb"] *= self.num_timesteps / 1000.0
+
+            target = {
+                ModelMeanType.PREVIOUS_X: self.q_posterior_mean_variance(
+                    x_start=x_start, x_t=x_t, t=t
+                )[0],
+                ModelMeanType.START_X: x_start,
+                ModelMeanType.EPSILON: noise,
+            }[self.model_mean_type]
+            assert model_output.shape == target.shape == x_start.shape
+            terms["mse"] = mean_flat((target - model_output) ** 2)
+            if "vb" in terms:
+                terms["loss"] = terms["mse"] + terms["vb"]
+            else:
+                terms["loss"] = terms["mse"]
+        else:
+            raise NotImplementedError(self.loss_type)
+
+        return terms
+
+    def _prior_bpd(self, x_start):
+        """
+        Get the prior KL term for the variational lower-bound, measured in
+        bits-per-dim.
+
+        This term can't be optimized, as it only depends on the encoder.
+
+        :param x_start: the [N x C x ...] tensor of inputs.
+        :return: a batch of [N] KL values (in bits), one per batch element.
+        """
+        batch_size = x_start.shape[0]
+        t = th.tensor([self.num_timesteps - 1] * batch_size, device=x_start.device)
+        qt_mean, _, qt_log_variance = self.q_mean_variance(x_start, t)
+        kl_prior = normal_kl(
+            mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0
+        )
+        return mean_flat(kl_prior) / np.log(2.0)
+
+    def calc_bpd_loop(self, model, x_start, clip_denoised=True, model_kwargs=None):
+        """
+        Compute the entire variational lower-bound, measured in bits-per-dim,
+        as well as other related quantities.
+
+        :param model: the model to evaluate loss on.
+        :param x_start: the [N x C x ...] tensor of inputs.
+        :param clip_denoised: if True, clip denoised samples.
+        :param model_kwargs: if not None, a dict of extra keyword arguments to
+            pass to the model. This can be used for conditioning.
+
+        :return: a dict containing the following keys:
+                 - total_bpd: the total variational lower-bound, per batch element.
+                 - prior_bpd: the prior term in the lower-bound.
+                 - vb: an [N x T] tensor of terms in the lower-bound.
+                 - xstart_mse: an [N x T] tensor of x_0 MSEs for each timestep.
+                 - mse: an [N x T] tensor of epsilon MSEs for each timestep.
+        """
+        device = x_start.device
+        batch_size = x_start.shape[0]
+
+        vb = []
+        xstart_mse = []
+        mse = []
+        for t in list(range(self.num_timesteps))[::-1]:
+            t_batch = th.tensor([t] * batch_size, device=device)
+            noise = th.randn_like(x_start)
+            x_t = self.q_sample(x_start=x_start, t=t_batch, noise=noise)
+            # Calculate VLB term at the current timestep
+            with th.no_grad():
+                out = self._vb_terms_bpd(
+                    model,
+                    x_start=x_start,
+                    x_t=x_t,
+                    t=t_batch,
+                    clip_denoised=clip_denoised,
+                    model_kwargs=model_kwargs,
+                )
+            vb.append(out["output"])
+            xstart_mse.append(mean_flat((out["pred_xstart"] - x_start) ** 2))
+            eps = self._predict_eps_from_xstart(x_t, t_batch, out["pred_xstart"])
+            mse.append(mean_flat((eps - noise) ** 2))
+
+        vb = th.stack(vb, dim=1)
+        xstart_mse = th.stack(xstart_mse, dim=1)
+        mse = th.stack(mse, dim=1)
+
+        prior_bpd = self._prior_bpd(x_start)
+        total_bpd = vb.sum(dim=1) + prior_bpd
+        return {
+            "total_bpd": total_bpd,
+            "prior_bpd": prior_bpd,
+            "vb": vb,
+            "xstart_mse": xstart_mse,
+            "mse": mse,
+        }
+
+
+def _extract_into_tensor(arr, timesteps, broadcast_shape):
+    """
+    Extract values from a 1-D numpy array for a batch of indices.
+
+    :param arr: the 1-D numpy array.
+    :param timesteps: a tensor of indices into the array to extract.
+    :param broadcast_shape: a larger shape of K dimensions with the batch
+                            dimension equal to the length of timesteps.
+    :return: a tensor of shape [batch_size, 1, ...] where the shape has K dims.
+    """
+    res = th.from_numpy(arr).to(device=timesteps.device)[timesteps].float()
+    while len(res.shape) < len(broadcast_shape):
+        res = res[..., None]
+    return res.expand(broadcast_shape)

guided_diffusion/losses.py

@@ -0,0 +1,77 @@
+"""
+Helpers for various likelihood-based losses. These are ported from the original
+Ho et al. diffusion models codebase:
+https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/utils.py
+"""
+
+import numpy as np
+
+import torch as th
+
+
+def normal_kl(mean1, logvar1, mean2, logvar2):
+    """
+    Compute the KL divergence between two gaussians.
+
+    Shapes are automatically broadcasted, so batches can be compared to
+    scalars, among other use cases.
+    """
+    tensor = None
+    for obj in (mean1, logvar1, mean2, logvar2):
+        if isinstance(obj, th.Tensor):
+            tensor = obj
+            break
+    assert tensor is not None, "at least one argument must be a Tensor"
+
+    # Force variances to be Tensors. Broadcasting helps convert scalars to
+    # Tensors, but it does not work for th.exp().
+    logvar1, logvar2 = [
+        x if isinstance(x, th.Tensor) else th.tensor(x).to(tensor)
+        for x in (logvar1, logvar2)
+    ]
+
+    return 0.5 * (
+        -1.0
+        + logvar2
+        - logvar1
+        + th.exp(logvar1 - logvar2)
+        + ((mean1 - mean2) ** 2) * th.exp(-logvar2)
+    )
+
+
+def approx_standard_normal_cdf(x):
+    """
+    A fast approximation of the cumulative distribution function of the
+    standard normal.
+    """
+    return 0.5 * (1.0 + th.tanh(np.sqrt(2.0 / np.pi) * (x + 0.044715 * th.pow(x, 3))))
+
+
+def discretized_gaussian_log_likelihood(x, *, means, log_scales):
+    """
+    Compute the log-likelihood of a Gaussian distribution discretizing to a
+    given image.
+
+    :param x: the target images. It is assumed that this was uint8 values,
+              rescaled to the range [-1, 1].
+    :param means: the Gaussian mean Tensor.
+    :param log_scales: the Gaussian log stddev Tensor.
+    :return: a tensor like x of log probabilities (in nats).
+    """
+    assert x.shape == means.shape == log_scales.shape
+    centered_x = x - means
+    inv_stdv = th.exp(-log_scales)
+    plus_in = inv_stdv * (centered_x + 1.0 / 255.0)
+    cdf_plus = approx_standard_normal_cdf(plus_in)
+    min_in = inv_stdv * (centered_x - 1.0 / 255.0)
+    cdf_min = approx_standard_normal_cdf(min_in)
+    log_cdf_plus = th.log(cdf_plus.clamp(min=1e-12))
+    log_one_minus_cdf_min = th.log((1.0 - cdf_min).clamp(min=1e-12))
+    cdf_delta = cdf_plus - cdf_min
+    log_probs = th.where(
+        x < -0.999,
+        log_cdf_plus,
+        th.where(x > 0.999, log_one_minus_cdf_min, th.log(cdf_delta.clamp(min=1e-12))),
+    )
+    assert log_probs.shape == x.shape
+    return log_probs

guided_diffusion/nn.py

@@ -0,0 +1,170 @@
+"""
+Various utilities for neural networks.
+"""
+
+import math
+
+import torch as th
+import torch.nn as nn
+
+
+# PyTorch 1.7 has SiLU, but we support PyTorch 1.5.
+class SiLU(nn.Module):
+    def forward(self, x):
+        return x * th.sigmoid(x)
+
+
+class GroupNorm32(nn.GroupNorm):
+    def forward(self, x):
+        return super().forward(x.float()).type(x.dtype)
+
+
+def conv_nd(dims, *args, **kwargs):
+    """
+    Create a 1D, 2D, or 3D convolution module.
+    """
+    if dims == 1:
+        return nn.Conv1d(*args, **kwargs)
+    elif dims == 2:
+        return nn.Conv2d(*args, **kwargs)
+    elif dims == 3:
+        return nn.Conv3d(*args, **kwargs)
+    raise ValueError(f"unsupported dimensions: {dims}")
+
+
+def linear(*args, **kwargs):
+    """
+    Create a linear module.
+    """
+    return nn.Linear(*args, **kwargs)
+
+
+def avg_pool_nd(dims, *args, **kwargs):
+    """
+    Create a 1D, 2D, or 3D average pooling module.
+    """
+    if dims == 1:
+        return nn.AvgPool1d(*args, **kwargs)
+    elif dims == 2:
+        return nn.AvgPool2d(*args, **kwargs)
+    elif dims == 3:
+        return nn.AvgPool3d(*args, **kwargs)
+    raise ValueError(f"unsupported dimensions: {dims}")
+
+
+def update_ema(target_params, source_params, rate=0.99):
+    """
+    Update target parameters to be closer to those of source parameters using
+    an exponential moving average.
+
+    :param target_params: the target parameter sequence.
+    :param source_params: the source parameter sequence.
+    :param rate: the EMA rate (closer to 1 means slower).
+    """
+    for targ, src in zip(target_params, source_params):
+        targ.detach().mul_(rate).add_(src, alpha=1 - rate)
+
+
+def zero_module(module):
+    """
+    Zero out the parameters of a module and return it.
+    """
+    for p in module.parameters():
+        p.detach().zero_()
+    return module
+
+
+def scale_module(module, scale):
+    """
+    Scale the parameters of a module and return it.
+    """
+    for p in module.parameters():
+        p.detach().mul_(scale)
+    return module
+
+
+def mean_flat(tensor):
+    """
+    Take the mean over all non-batch dimensions.
+    """
+    return tensor.mean(dim=list(range(1, len(tensor.shape))))
+
+
+def normalization(channels):
+    """
+    Make a standard normalization layer.
+
+    :param channels: number of input channels.
+    :return: an nn.Module for normalization.
+    """
+    return GroupNorm32(32, channels)
+
+
+def timestep_embedding(timesteps, dim, max_period=10000):
+    """
+    Create sinusoidal timestep embeddings.
+
+    :param timesteps: a 1-D Tensor of N indices, one per batch element.
+                      These may be fractional.
+    :param dim: the dimension of the output.
+    :param max_period: controls the minimum frequency of the embeddings.
+    :return: an [N x dim] Tensor of positional embeddings.
+    """
+    half = dim // 2
+    freqs = th.exp(
+        -math.log(max_period) * th.arange(start=0, end=half, dtype=th.float32) / half
+    ).to(device=timesteps.device)
+    args = timesteps[:, None].float() * freqs[None]
+    embedding = th.cat([th.cos(args), th.sin(args)], dim=-1)
+    if dim % 2:
+        embedding = th.cat([embedding, th.zeros_like(embedding[:, :1])], dim=-1)
+    return embedding
+
+
+def checkpoint(func, inputs, params, flag):
+    """
+    Evaluate a function without caching intermediate activations, allowing for
+    reduced memory at the expense of extra compute in the backward pass.
+
+    :param func: the function to evaluate.
+    :param inputs: the argument sequence to pass to `func`.
+    :param params: a sequence of parameters `func` depends on but does not
+                   explicitly take as arguments.
+    :param flag: if False, disable gradient checkpointing.
+    """
+    if flag:
+        args = tuple(inputs) + tuple(params)
+        return CheckpointFunction.apply(func, len(inputs), *args)
+    else:
+        return func(*inputs)
+
+
+class CheckpointFunction(th.autograd.Function):
+    @staticmethod
+    def forward(ctx, run_function, length, *args):
+        ctx.run_function = run_function
+        ctx.input_tensors = list(args[:length])
+        ctx.input_params = list(args[length:])
+        with th.no_grad():
+            output_tensors = ctx.run_function(*ctx.input_tensors)
+        return output_tensors
+
+    @staticmethod
+    def backward(ctx, *output_grads):
+        ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
+        with th.enable_grad():
+            # Fixes a bug where the first op in run_function modifies the
+            # Tensor storage in place, which is not allowed for detach()'d
+            # Tensors.
+            shallow_copies = [x.view_as(x) for x in ctx.input_tensors]
+            output_tensors = ctx.run_function(*shallow_copies)
+        input_grads = th.autograd.grad(
+            output_tensors,
+            ctx.input_tensors + ctx.input_params,
+            output_grads,
+            allow_unused=True,
+        )
+        del ctx.input_tensors
+        del ctx.input_params
+        del output_tensors
+        return (None, None) + input_grads

guided_diffusion/respace.py

@@ -0,0 +1,128 @@
+import numpy as np
+import torch as th
+
+from .gaussian_diffusion import GaussianDiffusion
+
+
+def space_timesteps(num_timesteps, section_counts):
+    """
+    Create a list of timesteps to use from an original diffusion process,
+    given the number of timesteps we want to take from equally-sized portions
+    of the original process.
+
+    For example, if there's 300 timesteps and the section counts are [10,15,20]
+    then the first 100 timesteps are strided to be 10 timesteps, the second 100
+    are strided to be 15 timesteps, and the final 100 are strided to be 20.
+
+    If the stride is a string starting with "ddim", then the fixed striding
+    from the DDIM paper is used, and only one section is allowed.
+
+    :param num_timesteps: the number of diffusion steps in the original
+                          process to divide up.
+    :param section_counts: either a list of numbers, or a string containing
+                           comma-separated numbers, indicating the step count
+                           per section. As a special case, use "ddimN" where N
+                           is a number of steps to use the striding from the
+                           DDIM paper.
+    :return: a set of diffusion steps from the original process to use.
+    """
+    if isinstance(section_counts, str):
+        if section_counts.startswith("ddim"):
+            desired_count = int(section_counts[len("ddim") :])
+            for i in range(1, num_timesteps):
+                if len(range(0, num_timesteps, i)) == desired_count:
+                    return set(range(0, num_timesteps, i))
+            raise ValueError(
+                f"cannot create exactly {num_timesteps} steps with an integer stride"
+            )
+        section_counts = [int(x) for x in section_counts.split(",")]
+    size_per = num_timesteps // len(section_counts)
+    extra = num_timesteps % len(section_counts)
+    start_idx = 0
+    all_steps = []
+    for i, section_count in enumerate(section_counts):
+        size = size_per + (1 if i < extra else 0)
+        if size < section_count:
+            raise ValueError(
+                f"cannot divide section of {size} steps into {section_count}"
+            )
+        if section_count <= 1:
+            frac_stride = 1
+        else:
+            frac_stride = (size - 1) / (section_count - 1)
+        cur_idx = 0.0
+        taken_steps = []
+        for _ in range(section_count):
+            taken_steps.append(start_idx + round(cur_idx))
+            cur_idx += frac_stride
+        all_steps += taken_steps
+        start_idx += size
+    return set(all_steps)
+
+
+class SpacedDiffusion(GaussianDiffusion):
+    """
+    A diffusion process which can skip steps in a base diffusion process.
+
+    :param use_timesteps: a collection (sequence or set) of timesteps from the
+                          original diffusion process to retain.
+    :param kwargs: the kwargs to create the base diffusion process.
+    """
+
+    def __init__(self, use_timesteps, **kwargs):
+        self.use_timesteps = set(use_timesteps)
+        self.timestep_map = []
+        self.original_num_steps = len(kwargs["betas"])
+
+        base_diffusion = GaussianDiffusion(**kwargs)  # pylint: disable=missing-kwoa
+        last_alpha_cumprod = 1.0
+        new_betas = []
+        for i, alpha_cumprod in enumerate(base_diffusion.alphas_cumprod):
+            if i in self.use_timesteps:
+                new_betas.append(1 - alpha_cumprod / last_alpha_cumprod)
+                last_alpha_cumprod = alpha_cumprod
+                self.timestep_map.append(i)
+        kwargs["betas"] = np.array(new_betas)
+        super().__init__(**kwargs)
+
+    def p_mean_variance(
+        self, model, *args, **kwargs
+    ):  # pylint: disable=signature-differs
+        return super().p_mean_variance(self._wrap_model(model), *args, **kwargs)
+
+    def training_losses(
+        self, model, *args, **kwargs
+    ):  # pylint: disable=signature-differs
+        return super().training_losses(self._wrap_model(model), *args, **kwargs)
+
+    def condition_mean(self, cond_fn, *args, **kwargs):
+        return super().condition_mean(self._wrap_model(cond_fn), *args, **kwargs)
+
+    def condition_score(self, cond_fn, *args, **kwargs):
+        return super().condition_score(self._wrap_model(cond_fn), *args, **kwargs)
+
+    def _wrap_model(self, model):
+        if isinstance(model, _WrappedModel):
+            return model
+        return _WrappedModel(
+            model, self.timestep_map, self.rescale_timesteps, self.original_num_steps
+        )
+
+    def _scale_timesteps(self, t):
+        # Scaling is done by the wrapped model.
+        return t
+
+
+class _WrappedModel:
+    def __init__(self, model, timestep_map, rescale_timesteps, original_num_steps):
+        self.model = model
+        self.timestep_map = timestep_map
+        self.rescale_timesteps = rescale_timesteps
+        self.original_num_steps = original_num_steps
+
+    def __call__(self, x, ts, **kwargs):
+        map_tensor = th.tensor(self.timestep_map, device=ts.device, dtype=ts.dtype)
+        new_ts = map_tensor[ts]
+        if self.rescale_timesteps:
+            new_ts = new_ts.float() * (1000.0 / self.original_num_steps)
+        return self.model(x, new_ts, **kwargs)

guided_diffusion/script_util.py

@@ -0,0 +1,452 @@
+import argparse
+import inspect
+
+from . import gaussian_diffusion as gd
+from .respace import SpacedDiffusion, space_timesteps
+from .unet import SuperResModel, UNetModel, EncoderUNetModel
+
+NUM_CLASSES = 1000
+
+
+def diffusion_defaults():
+    """
+    Defaults for image and classifier training.
+    """
+    return dict(
+        learn_sigma=False,
+        diffusion_steps=1000,
+        noise_schedule="linear",
+        timestep_respacing="",
+        use_kl=False,
+        predict_xstart=False,
+        rescale_timesteps=False,
+        rescale_learned_sigmas=False,
+    )
+
+
+def classifier_defaults():
+    """
+    Defaults for classifier models.
+    """
+    return dict(
+        image_size=64,
+        classifier_use_fp16=False,
+        classifier_width=128,
+        classifier_depth=2,
+        classifier_attention_resolutions="32,16,8",  # 16
+        classifier_use_scale_shift_norm=True,  # False
+        classifier_resblock_updown=True,  # False
+        classifier_pool="attention",
+    )
+
+
+def model_and_diffusion_defaults():
+    """
+    Defaults for image training.
+    """
+    res = dict(
+        image_size=64,
+        num_channels=128,
+        num_res_blocks=2,
+        num_heads=4,
+        num_heads_upsample=-1,
+        num_head_channels=-1,
+        attention_resolutions="16,8",
+        channel_mult="",
+        dropout=0.0,
+        class_cond=False,
+        use_checkpoint=False,
+        use_scale_shift_norm=True,
+        resblock_updown=False,
+        use_fp16=False,
+        use_new_attention_order=False,
+    )
+    res.update(diffusion_defaults())
+    return res
+
+
+def classifier_and_diffusion_defaults():
+    res = classifier_defaults()
+    res.update(diffusion_defaults())
+    return res
+
+
+def create_model_and_diffusion(
+    image_size,
+    class_cond,
+    learn_sigma,
+    num_channels,
+    num_res_blocks,
+    channel_mult,
+    num_heads,
+    num_head_channels,
+    num_heads_upsample,
+    attention_resolutions,
+    dropout,
+    diffusion_steps,
+    noise_schedule,
+    timestep_respacing,
+    use_kl,
+    predict_xstart,
+    rescale_timesteps,
+    rescale_learned_sigmas,
+    use_checkpoint,
+    use_scale_shift_norm,
+    resblock_updown,
+    use_fp16,
+    use_new_attention_order,
+):
+    model = create_model(
+        image_size,
+        num_channels,
+        num_res_blocks,
+        channel_mult=channel_mult,
+        learn_sigma=learn_sigma,
+        class_cond=class_cond,
+        use_checkpoint=use_checkpoint,
+        attention_resolutions=attention_resolutions,
+        num_heads=num_heads,
+        num_head_channels=num_head_channels,
+        num_heads_upsample=num_heads_upsample,
+        use_scale_shift_norm=use_scale_shift_norm,
+        dropout=dropout,
+        resblock_updown=resblock_updown,
+        use_fp16=use_fp16,
+        use_new_attention_order=use_new_attention_order,
+    )
+    diffusion = create_gaussian_diffusion(
+        steps=diffusion_steps,
+        learn_sigma=learn_sigma,
+        noise_schedule=noise_schedule,
+        use_kl=use_kl,
+        predict_xstart=predict_xstart,
+        rescale_timesteps=rescale_timesteps,
+        rescale_learned_sigmas=rescale_learned_sigmas,
+        timestep_respacing=timestep_respacing,
+    )
+    return model, diffusion
+
+
+def create_model(
+    image_size,
+    num_channels,
+    num_res_blocks,
+    channel_mult="",
+    learn_sigma=False,
+    class_cond=False,
+    use_checkpoint=False,
+    attention_resolutions="16",
+    num_heads=1,
+    num_head_channels=-1,
+    num_heads_upsample=-1,
+    use_scale_shift_norm=False,
+    dropout=0,
+    resblock_updown=False,
+    use_fp16=False,
+    use_new_attention_order=False,
+):
+    if channel_mult == "":
+        if image_size == 512:
+            channel_mult = (0.5, 1, 1, 2, 2, 4, 4)
+        elif image_size == 256:
+            channel_mult = (1, 1, 2, 2, 4, 4)
+        elif image_size == 128:
+            channel_mult = (1, 1, 2, 3, 4)
+        elif image_size == 64:
+            channel_mult = (1, 2, 3, 4)
+        else:
+            raise ValueError(f"unsupported image size: {image_size}")
+    else:
+        channel_mult = tuple(int(ch_mult) for ch_mult in channel_mult.split(","))
+
+    attention_ds = []
+    for res in attention_resolutions.split(","):
+        attention_ds.append(image_size // int(res))
+
+    return UNetModel(
+        image_size=image_size,
+        in_channels=3,
+        model_channels=num_channels,
+        out_channels=(3 if not learn_sigma else 6),
+        num_res_blocks=num_res_blocks,
+        attention_resolutions=tuple(attention_ds),
+        dropout=dropout,
+        channel_mult=channel_mult,
+        num_classes=(NUM_CLASSES if class_cond else None),
+        use_checkpoint=use_checkpoint,
+        use_fp16=use_fp16,
+        num_heads=num_heads,
+        num_head_channels=num_head_channels,
+        num_heads_upsample=num_heads_upsample,
+        use_scale_shift_norm=use_scale_shift_norm,
+        resblock_updown=resblock_updown,
+        use_new_attention_order=use_new_attention_order,
+    )
+
+
+def create_classifier_and_diffusion(
+    image_size,
+    classifier_use_fp16,
+    classifier_width,
+    classifier_depth,
+    classifier_attention_resolutions,
+    classifier_use_scale_shift_norm,
+    classifier_resblock_updown,
+    classifier_pool,
+    learn_sigma,
+    diffusion_steps,
+    noise_schedule,
+    timestep_respacing,
+    use_kl,
+    predict_xstart,
+    rescale_timesteps,
+    rescale_learned_sigmas,
+):
+    classifier = create_classifier(
+        image_size,
+        classifier_use_fp16,
+        classifier_width,
+        classifier_depth,
+        classifier_attention_resolutions,
+        classifier_use_scale_shift_norm,
+        classifier_resblock_updown,
+        classifier_pool,
+    )
+    diffusion = create_gaussian_diffusion(
+        steps=diffusion_steps,
+        learn_sigma=learn_sigma,
+        noise_schedule=noise_schedule,
+        use_kl=use_kl,
+        predict_xstart=predict_xstart,
+        rescale_timesteps=rescale_timesteps,
+        rescale_learned_sigmas=rescale_learned_sigmas,
+        timestep_respacing=timestep_respacing,
+    )
+    return classifier, diffusion
+
+
+def create_classifier(
+    image_size,
+    classifier_use_fp16,
+    classifier_width,
+    classifier_depth,
+    classifier_attention_resolutions,
+    classifier_use_scale_shift_norm,
+    classifier_resblock_updown,
+    classifier_pool,
+):
+    if image_size == 512:
+        channel_mult = (0.5, 1, 1, 2, 2, 4, 4)
+    elif image_size == 256:
+        channel_mult = (1, 1, 2, 2, 4, 4)
+    elif image_size == 128:
+        channel_mult = (1, 1, 2, 3, 4)
+    elif image_size == 64:
+        channel_mult = (1, 2, 3, 4)
+    else:
+        raise ValueError(f"unsupported image size: {image_size}")
+
+    attention_ds = []
+    for res in classifier_attention_resolutions.split(","):
+        attention_ds.append(image_size // int(res))
+
+    return EncoderUNetModel(
+        image_size=image_size,
+        in_channels=3,
+        model_channels=classifier_width,
+        out_channels=1000,
+        num_res_blocks=classifier_depth,
+        attention_resolutions=tuple(attention_ds),
+        channel_mult=channel_mult,
+        use_fp16=classifier_use_fp16,
+        num_head_channels=64,
+        use_scale_shift_norm=classifier_use_scale_shift_norm,
+        resblock_updown=classifier_resblock_updown,
+        pool=classifier_pool,
+    )
+
+
+def sr_model_and_diffusion_defaults():
+    res = model_and_diffusion_defaults()
+    res["large_size"] = 256
+    res["small_size"] = 64
+    arg_names = inspect.getfullargspec(sr_create_model_and_diffusion)[0]
+    for k in res.copy().keys():
+        if k not in arg_names:
+            del res[k]
+    return res
+
+
+def sr_create_model_and_diffusion(
+    large_size,
+    small_size,
+    class_cond,
+    learn_sigma,
+    num_channels,
+    num_res_blocks,
+    num_heads,
+    num_head_channels,
+    num_heads_upsample,
+    attention_resolutions,
+    dropout,
+    diffusion_steps,
+    noise_schedule,
+    timestep_respacing,
+    use_kl,
+    predict_xstart,
+    rescale_timesteps,
+    rescale_learned_sigmas,
+    use_checkpoint,
+    use_scale_shift_norm,
+    resblock_updown,
+    use_fp16,
+):
+    model = sr_create_model(
+        large_size,
+        small_size,
+        num_channels,
+        num_res_blocks,
+        learn_sigma=learn_sigma,
+        class_cond=class_cond,
+        use_checkpoint=use_checkpoint,
+        attention_resolutions=attention_resolutions,
+        num_heads=num_heads,
+        num_head_channels=num_head_channels,
+        num_heads_upsample=num_heads_upsample,
+        use_scale_shift_norm=use_scale_shift_norm,
+        dropout=dropout,
+        resblock_updown=resblock_updown,
+        use_fp16=use_fp16,
+    )
+    diffusion = create_gaussian_diffusion(
+        steps=diffusion_steps,
+        learn_sigma=learn_sigma,
+        noise_schedule=noise_schedule,
+        use_kl=use_kl,
+        predict_xstart=predict_xstart,
+        rescale_timesteps=rescale_timesteps,
+        rescale_learned_sigmas=rescale_learned_sigmas,
+        timestep_respacing=timestep_respacing,
+    )
+    return model, diffusion
+
+
+def sr_create_model(
+    large_size,
+    small_size,
+    num_channels,
+    num_res_blocks,
+    learn_sigma,
+    class_cond,
+    use_checkpoint,
+    attention_resolutions,
+    num_heads,
+    num_head_channels,
+    num_heads_upsample,
+    use_scale_shift_norm,
+    dropout,
+    resblock_updown,
+    use_fp16,
+):
+    _ = small_size  # hack to prevent unused variable
+
+    if large_size == 512:
+        channel_mult = (1, 1, 2, 2, 4, 4)
+    elif large_size == 256:
+        channel_mult = (1, 1, 2, 2, 4, 4)
+    elif large_size == 64:
+        channel_mult = (1, 2, 3, 4)
+    else:
+        raise ValueError(f"unsupported large size: {large_size}")
+
+    attention_ds = []
+    for res in attention_resolutions.split(","):
+        attention_ds.append(large_size // int(res))
+
+    return SuperResModel(
+        image_size=large_size,
+        in_channels=3,
+        model_channels=num_channels,
+        out_channels=(3 if not learn_sigma else 6),
+        num_res_blocks=num_res_blocks,
+        attention_resolutions=tuple(attention_ds),
+        dropout=dropout,
+        channel_mult=channel_mult,
+        num_classes=(NUM_CLASSES if class_cond else None),
+        use_checkpoint=use_checkpoint,
+        num_heads=num_heads,
+        num_head_channels=num_head_channels,
+        num_heads_upsample=num_heads_upsample,
+        use_scale_shift_norm=use_scale_shift_norm,
+        resblock_updown=resblock_updown,
+        use_fp16=use_fp16,
+    )
+
+
+def create_gaussian_diffusion(
+    *,
+    steps=1000,
+    learn_sigma=False,
+    sigma_small=False,
+    noise_schedule="linear",
+    use_kl=False,
+    predict_xstart=False,
+    rescale_timesteps=False,
+    rescale_learned_sigmas=False,
+    timestep_respacing="",
+):
+    betas = gd.get_named_beta_schedule(noise_schedule, steps)
+    if use_kl:
+        loss_type = gd.LossType.RESCALED_KL
+    elif rescale_learned_sigmas:
+        loss_type = gd.LossType.RESCALED_MSE
+    else:
+        loss_type = gd.LossType.MSE
+    if not timestep_respacing:
+        timestep_respacing = [steps]
+    return SpacedDiffusion(
+        use_timesteps=space_timesteps(steps, timestep_respacing),
+        betas=betas,
+        model_mean_type=(
+            gd.ModelMeanType.EPSILON if not predict_xstart else gd.ModelMeanType.START_X
+        ),
+        model_var_type=(
+            (
+                gd.ModelVarType.FIXED_LARGE
+                if not sigma_small
+                else gd.ModelVarType.FIXED_SMALL
+            )
+            if not learn_sigma
+            else gd.ModelVarType.LEARNED_RANGE
+        ),
+        loss_type=loss_type,
+        rescale_timesteps=rescale_timesteps,
+    )
+
+
+def add_dict_to_argparser(parser, default_dict):
+    for k, v in default_dict.items():
+        v_type = type(v)
+        if v is None:
+            v_type = str
+        elif isinstance(v, bool):
+            v_type = str2bool
+        parser.add_argument(f"--{k}", default=v, type=v_type)
+
+
+def args_to_dict(args, keys):
+    return {k: getattr(args, k) for k in keys}
+
+
+def str2bool(v):
+    """
+    https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse
+    """
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("boolean value expected")

guided_diffusion/unet.py

@@ -0,0 +1,894 @@
+from abc import abstractmethod
+
+import math
+
+import numpy as np
+import torch as th
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .fp16_util import convert_module_to_f16, convert_module_to_f32
+from .nn import (
+    checkpoint,
+    conv_nd,
+    linear,
+    avg_pool_nd,
+    zero_module,
+    normalization,
+    timestep_embedding,
+)
+
+
+class AttentionPool2d(nn.Module):
+    """
+    Adapted from CLIP: https://github.com/openai/CLIP/blob/main/clip/model.py
+    """
+
+    def __init__(
+        self,
+        spacial_dim: int,
+        embed_dim: int,
+        num_heads_channels: int,
+        output_dim: int = None,
+    ):
+        super().__init__()
+        self.positional_embedding = nn.Parameter(
+            th.randn(embed_dim, spacial_dim ** 2 + 1) / embed_dim ** 0.5
+        )
+        self.qkv_proj = conv_nd(1, embed_dim, 3 * embed_dim, 1)
+        self.c_proj = conv_nd(1, embed_dim, output_dim or embed_dim, 1)
+        self.num_heads = embed_dim // num_heads_channels
+        self.attention = QKVAttention(self.num_heads)
+
+    def forward(self, x):
+        b, c, *_spatial = x.shape
+        x = x.reshape(b, c, -1)  # NC(HW)
+        x = th.cat([x.mean(dim=-1, keepdim=True), x], dim=-1)  # NC(HW+1)
+        x = x + self.positional_embedding[None, :, :].to(x.dtype)  # NC(HW+1)
+        x = self.qkv_proj(x)
+        x = self.attention(x)
+        x = self.c_proj(x)
+        return x[:, :, 0]
+
+
+class TimestepBlock(nn.Module):
+    """
+    Any module where forward() takes timestep embeddings as a second argument.
+    """
+
+    @abstractmethod
+    def forward(self, x, emb):
+        """
+        Apply the module to `x` given `emb` timestep embeddings.
+        """
+
+
+class TimestepEmbedSequential(nn.Sequential, TimestepBlock):
+    """
+    A sequential module that passes timestep embeddings to the children that
+    support it as an extra input.
+    """
+
+    def forward(self, x, emb):
+        for layer in self:
+            if isinstance(layer, TimestepBlock):
+                x = layer(x, emb)
+            else:
+                x = layer(x)
+        return x
+
+
+class Upsample(nn.Module):
+    """
+    An upsampling layer with an optional convolution.
+
+    :param channels: channels in the inputs and outputs.
+    :param use_conv: a bool determining if a convolution is applied.
+    :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then
+                 upsampling occurs in the inner-two dimensions.
+    """
+
+    def __init__(self, channels, use_conv, dims=2, out_channels=None):
+        super().__init__()
+        self.channels = channels
+        self.out_channels = out_channels or channels
+        self.use_conv = use_conv
+        self.dims = dims
+        if use_conv:
+            self.conv = conv_nd(dims, self.channels, self.out_channels, 3, padding=1)
+
+    def forward(self, x):
+        assert x.shape[1] == self.channels
+        if self.dims == 3:
+            x = F.interpolate(
+                x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), mode="nearest"
+            )
+        else:
+            x = F.interpolate(x, scale_factor=2, mode="nearest")
+        if self.use_conv:
+            x = self.conv(x)
+        return x
+
+
+class Downsample(nn.Module):
+    """
+    A downsampling layer with an optional convolution.
+
+    :param channels: channels in the inputs and outputs.
+    :param use_conv: a bool determining if a convolution is applied.
+    :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then
+                 downsampling occurs in the inner-two dimensions.
+    """
+
+    def __init__(self, channels, use_conv, dims=2, out_channels=None):
+        super().__init__()
+        self.channels = channels
+        self.out_channels = out_channels or channels
+        self.use_conv = use_conv
+        self.dims = dims
+        stride = 2 if dims != 3 else (1, 2, 2)
+        if use_conv:
+            self.op = conv_nd(
+                dims, self.channels, self.out_channels, 3, stride=stride, padding=1
+            )
+        else:
+            assert self.channels == self.out_channels
+            self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride)
+
+    def forward(self, x):
+        assert x.shape[1] == self.channels
+        return self.op(x)
+
+
+class ResBlock(TimestepBlock):
+    """
+    A residual block that can optionally change the number of channels.
+
+    :param channels: the number of input channels.
+    :param emb_channels: the number of timestep embedding channels.
+    :param dropout: the rate of dropout.
+    :param out_channels: if specified, the number of out channels.
+    :param use_conv: if True and out_channels is specified, use a spatial
+        convolution instead of a smaller 1x1 convolution to change the
+        channels in the skip connection.
+    :param dims: determines if the signal is 1D, 2D, or 3D.
+    :param use_checkpoint: if True, use gradient checkpointing on this module.
+    :param up: if True, use this block for upsampling.
+    :param down: if True, use this block for downsampling.
+    """
+
+    def __init__(
+        self,
+        channels,
+        emb_channels,
+        dropout,
+        out_channels=None,
+        use_conv=False,
+        use_scale_shift_norm=False,
+        dims=2,
+        use_checkpoint=False,
+        up=False,
+        down=False,
+    ):
+        super().__init__()
+        self.channels = channels
+        self.emb_channels = emb_channels
+        self.dropout = dropout
+        self.out_channels = out_channels or channels
+        self.use_conv = use_conv
+        self.use_checkpoint = use_checkpoint
+        self.use_scale_shift_norm = use_scale_shift_norm
+
+        self.in_layers = nn.Sequential(
+            normalization(channels),
+            nn.SiLU(),
+            conv_nd(dims, channels, self.out_channels, 3, padding=1),
+        )
+
+        self.updown = up or down
+
+        if up:
+            self.h_upd = Upsample(channels, False, dims)
+            self.x_upd = Upsample(channels, False, dims)
+        elif down:
+            self.h_upd = Downsample(channels, False, dims)
+            self.x_upd = Downsample(channels, False, dims)
+        else:
+            self.h_upd = self.x_upd = nn.Identity()
+
+        self.emb_layers = nn.Sequential(
+            nn.SiLU(),
+            linear(
+                emb_channels,
+                2 * self.out_channels if use_scale_shift_norm else self.out_channels,
+            ),
+        )
+        self.out_layers = nn.Sequential(
+            normalization(self.out_channels),
+            nn.SiLU(),
+            nn.Dropout(p=dropout),
+            zero_module(
+                conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1)
+            ),
+        )
+
+        if self.out_channels == channels:
+            self.skip_connection = nn.Identity()
+        elif use_conv:
+            self.skip_connection = conv_nd(
+                dims, channels, self.out_channels, 3, padding=1
+            )
+        else:
+            self.skip_connection = conv_nd(dims, channels, self.out_channels, 1)
+
+    def forward(self, x, emb):
+        """
+        Apply the block to a Tensor, conditioned on a timestep embedding.
+
+        :param x: an [N x C x ...] Tensor of features.
+        :param emb: an [N x emb_channels] Tensor of timestep embeddings.
+        :return: an [N x C x ...] Tensor of outputs.
+        """
+        return checkpoint(
+            self._forward, (x, emb), self.parameters(), self.use_checkpoint
+        )
+
+    def _forward(self, x, emb):
+        if self.updown:
+            in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1]
+            h = in_rest(x)
+            h = self.h_upd(h)
+            x = self.x_upd(x)
+            h = in_conv(h)
+        else:
+            h = self.in_layers(x)
+        emb_out = self.emb_layers(emb).type(h.dtype)
+        while len(emb_out.shape) < len(h.shape):
+            emb_out = emb_out[..., None]
+        if self.use_scale_shift_norm:
+            out_norm, out_rest = self.out_layers[0], self.out_layers[1:]
+            scale, shift = th.chunk(emb_out, 2, dim=1)
+            h = out_norm(h) * (1 + scale) + shift
+            h = out_rest(h)
+        else:
+            h = h + emb_out
+            h = self.out_layers(h)
+        return self.skip_connection(x) + h
+
+
+class AttentionBlock(nn.Module):
+    """
+    An attention block that allows spatial positions to attend to each other.
+
+    Originally ported from here, but adapted to the N-d case.
+    https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/models/unet.py#L66.
+    """
+
+    def __init__(
+        self,
+        channels,
+        num_heads=1,
+        num_head_channels=-1,
+        use_checkpoint=False,
+        use_new_attention_order=False,
+    ):
+        super().__init__()
+        self.channels = channels
+        if num_head_channels == -1:
+            self.num_heads = num_heads
+        else:
+            assert (
+                channels % num_head_channels == 0
+            ), f"q,k,v channels {channels} is not divisible by num_head_channels {num_head_channels}"
+            self.num_heads = channels // num_head_channels
+        self.use_checkpoint = use_checkpoint
+        self.norm = normalization(channels)
+        self.qkv = conv_nd(1, channels, channels * 3, 1)
+        if use_new_attention_order:
+            # split qkv before split heads
+            self.attention = QKVAttention(self.num_heads)
+        else:
+            # split heads before split qkv
+            self.attention = QKVAttentionLegacy(self.num_heads)
+
+        self.proj_out = zero_module(conv_nd(1, channels, channels, 1))
+
+    def forward(self, x):
+        return checkpoint(self._forward, (x,), self.parameters(), True)
+
+    def _forward(self, x):
+        b, c, *spatial = x.shape
+        x = x.reshape(b, c, -1)
+        qkv = self.qkv(self.norm(x))
+        h = self.attention(qkv)
+        h = self.proj_out(h)
+        return (x + h).reshape(b, c, *spatial)
+
+
+def count_flops_attn(model, _x, y):
+    """
+    A counter for the `thop` package to count the operations in an
+    attention operation.
+    Meant to be used like:
+        macs, params = thop.profile(
+            model,
+            inputs=(inputs, timestamps),
+            custom_ops={QKVAttention: QKVAttention.count_flops},
+        )
+    """
+    b, c, *spatial = y[0].shape
+    num_spatial = int(np.prod(spatial))
+    # We perform two matmuls with the same number of ops.
+    # The first computes the weight matrix, the second computes
+    # the combination of the value vectors.
+    matmul_ops = 2 * b * (num_spatial ** 2) * c
+    model.total_ops += th.DoubleTensor([matmul_ops])
+
+
+class QKVAttentionLegacy(nn.Module):
+    """
+    A module which performs QKV attention. Matches legacy QKVAttention + input/ouput heads shaping
+    """
+
+    def __init__(self, n_heads):
+        super().__init__()
+        self.n_heads = n_heads
+
+    def forward(self, qkv):
+        """
+        Apply QKV attention.
+
+        :param qkv: an [N x (H * 3 * C) x T] tensor of Qs, Ks, and Vs.
+        :return: an [N x (H * C) x T] tensor after attention.
+        """
+        bs, width, length = qkv.shape
+        assert width % (3 * self.n_heads) == 0
+        ch = width // (3 * self.n_heads)
+        q, k, v = qkv.reshape(bs * self.n_heads, ch * 3, length).split(ch, dim=1)
+        scale = 1 / math.sqrt(math.sqrt(ch))
+        weight = th.einsum(
+            "bct,bcs->bts", q * scale, k * scale
+        )  # More stable with f16 than dividing afterwards
+        weight = th.softmax(weight.float(), dim=-1).type(weight.dtype)
+        a = th.einsum("bts,bcs->bct", weight, v)
+        return a.reshape(bs, -1, length)
+
+    @staticmethod
+    def count_flops(model, _x, y):
+        return count_flops_attn(model, _x, y)
+
+
+class QKVAttention(nn.Module):
+    """
+    A module which performs QKV attention and splits in a different order.
+    """
+
+    def __init__(self, n_heads):
+        super().__init__()
+        self.n_heads = n_heads
+
+    def forward(self, qkv):
+        """
+        Apply QKV attention.
+
+        :param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs.
+        :return: an [N x (H * C) x T] tensor after attention.
+        """
+        bs, width, length = qkv.shape
+        assert width % (3 * self.n_heads) == 0
+        ch = width // (3 * self.n_heads)
+        q, k, v = qkv.chunk(3, dim=1)
+        scale = 1 / math.sqrt(math.sqrt(ch))
+        weight = th.einsum(
+            "bct,bcs->bts",
+            (q * scale).view(bs * self.n_heads, ch, length),
+            (k * scale).view(bs * self.n_heads, ch, length),
+        )  # More stable with f16 than dividing afterwards
+        weight = th.softmax(weight.float(), dim=-1).type(weight.dtype)
+        a = th.einsum("bts,bcs->bct", weight, v.reshape(bs * self.n_heads, ch, length))
+        return a.reshape(bs, -1, length)
+
+    @staticmethod
+    def count_flops(model, _x, y):
+        return count_flops_attn(model, _x, y)
+
+
+class UNetModel(nn.Module):
+    """
+    The full UNet model with attention and timestep embedding.
+
+    :param in_channels: channels in the input Tensor.
+    :param model_channels: base channel count for the model.
+    :param out_channels: channels in the output Tensor.
+    :param num_res_blocks: number of residual blocks per downsample.
+    :param attention_resolutions: a collection of downsample rates at which
+        attention will take place. May be a set, list, or tuple.
+        For example, if this contains 4, then at 4x downsampling, attention
+        will be used.
+    :param dropout: the dropout probability.
+    :param channel_mult: channel multiplier for each level of the UNet.
+    :param conv_resample: if True, use learned convolutions for upsampling and
+        downsampling.
+    :param dims: determines if the signal is 1D, 2D, or 3D.
+    :param num_classes: if specified (as an int), then this model will be
+        class-conditional with `num_classes` classes.
+    :param use_checkpoint: use gradient checkpointing to reduce memory usage.
+    :param num_heads: the number of attention heads in each attention layer.
+    :param num_heads_channels: if specified, ignore num_heads and instead use
+                               a fixed channel width per attention head.
+    :param num_heads_upsample: works with num_heads to set a different number
+                               of heads for upsampling. Deprecated.
+    :param use_scale_shift_norm: use a FiLM-like conditioning mechanism.
+    :param resblock_updown: use residual blocks for up/downsampling.
+    :param use_new_attention_order: use a different attention pattern for potentially
+                                    increased efficiency.
+    """
+
+    def __init__(
+        self,
+        image_size,
+        in_channels,
+        model_channels,
+        out_channels,
+        num_res_blocks,
+        attention_resolutions,
+        dropout=0,
+        channel_mult=(1, 2, 4, 8),
+        conv_resample=True,
+        dims=2,
+        num_classes=None,
+        use_checkpoint=False,
+        use_fp16=False,
+        num_heads=1,
+        num_head_channels=-1,
+        num_heads_upsample=-1,
+        use_scale_shift_norm=False,
+        resblock_updown=False,
+        use_new_attention_order=False,
+    ):
+        super().__init__()
+
+        if num_heads_upsample == -1:
+            num_heads_upsample = num_heads
+
+        self.image_size = image_size
+        self.in_channels = in_channels
+        self.model_channels = model_channels
+        self.out_channels = out_channels
+        self.num_res_blocks = num_res_blocks
+        self.attention_resolutions = attention_resolutions
+        self.dropout = dropout
+        self.channel_mult = channel_mult
+        self.conv_resample = conv_resample
+        self.num_classes = num_classes
+        self.use_checkpoint = use_checkpoint
+        self.dtype = th.float16 if use_fp16 else th.float32
+        self.num_heads = num_heads
+        self.num_head_channels = num_head_channels
+        self.num_heads_upsample = num_heads_upsample
+
+        time_embed_dim = model_channels * 4
+        self.time_embed = nn.Sequential(
+            linear(model_channels, time_embed_dim),
+            nn.SiLU(),
+            linear(time_embed_dim, time_embed_dim),
+        )
+
+        if self.num_classes is not None:
+            self.label_emb = nn.Embedding(num_classes, time_embed_dim)
+
+        ch = input_ch = int(channel_mult[0] * model_channels)
+        self.input_blocks = nn.ModuleList(
+            [TimestepEmbedSequential(conv_nd(dims, in_channels, ch, 3, padding=1))]
+        )
+        self._feature_size = ch
+        input_block_chans = [ch]
+        ds = 1
+        for level, mult in enumerate(channel_mult):
+            for _ in range(num_res_blocks):
+                layers = [
+                    ResBlock(
+                        ch,
+                        time_embed_dim,
+                        dropout,
+                        out_channels=int(mult * model_channels),
+                        dims=dims,
+                        use_checkpoint=use_checkpoint,
+                        use_scale_shift_norm=use_scale_shift_norm,
+                    )
+                ]
+                ch = int(mult * model_channels)
+                if ds in attention_resolutions:
+                    layers.append(
+                        AttentionBlock(
+                            ch,
+                            use_checkpoint=use_checkpoint,
+                            num_heads=num_heads,
+                            num_head_channels=num_head_channels,
+                            use_new_attention_order=use_new_attention_order,
+                        )
+                    )
+                self.input_blocks.append(TimestepEmbedSequential(*layers))
+                self._feature_size += ch
+                input_block_chans.append(ch)
+            if level != len(channel_mult) - 1:
+                out_ch = ch
+                self.input_blocks.append(
+                    TimestepEmbedSequential(
+                        ResBlock(
+                            ch,
+                            time_embed_dim,
+                            dropout,
+                            out_channels=out_ch,
+                            dims=dims,
+                            use_checkpoint=use_checkpoint,
+                            use_scale_shift_norm=use_scale_shift_norm,
+                            down=True,
+                        )
+                        if resblock_updown
+                        else Downsample(
+                            ch, conv_resample, dims=dims, out_channels=out_ch
+                        )
+                    )
+                )
+                ch = out_ch
+                input_block_chans.append(ch)
+                ds *= 2
+                self._feature_size += ch
+
+        self.middle_block = TimestepEmbedSequential(
+            ResBlock(
+                ch,
+                time_embed_dim,
+                dropout,
+                dims=dims,
+                use_checkpoint=use_checkpoint,
+                use_scale_shift_norm=use_scale_shift_norm,
+            ),
+            AttentionBlock(
+                ch,
+                use_checkpoint=use_checkpoint,
+                num_heads=num_heads,
+                num_head_channels=num_head_channels,
+                use_new_attention_order=use_new_attention_order,
+            ),
+            ResBlock(
+                ch,
+                time_embed_dim,
+                dropout,
+                dims=dims,
+                use_checkpoint=use_checkpoint,
+                use_scale_shift_norm=use_scale_shift_norm,
+            ),
+        )
+        self._feature_size += ch
+
+        self.output_blocks = nn.ModuleList([])
+        for level, mult in list(enumerate(channel_mult))[::-1]:
+            for i in range(num_res_blocks + 1):
+                ich = input_block_chans.pop()
+                layers = [
+                    ResBlock(
+                        ch + ich,
+                        time_embed_dim,
+                        dropout,
+                        out_channels=int(model_channels * mult),
+                        dims=dims,
+                        use_checkpoint=use_checkpoint,
+                        use_scale_shift_norm=use_scale_shift_norm,
+                    )
+                ]
+                ch = int(model_channels * mult)
+                if ds in attention_resolutions:
+                    layers.append(
+                        AttentionBlock(
+                            ch,
+                            use_checkpoint=use_checkpoint,
+                            num_heads=num_heads_upsample,
+                            num_head_channels=num_head_channels,
+                            use_new_attention_order=use_new_attention_order,
+                        )
+                    )
+                if level and i == num_res_blocks:
+                    out_ch = ch
+                    layers.append(
+                        ResBlock(
+                            ch,
+                            time_embed_dim,
+                            dropout,
+                            out_channels=out_ch,
+                            dims=dims,
+                            use_checkpoint=use_checkpoint,
+                            use_scale_shift_norm=use_scale_shift_norm,
+                            up=True,
+                        )
+                        if resblock_updown
+                        else Upsample(ch, conv_resample, dims=dims, out_channels=out_ch)
+                    )
+                    ds //= 2
+                self.output_blocks.append(TimestepEmbedSequential(*layers))
+                self._feature_size += ch
+
+        self.out = nn.Sequential(
+            normalization(ch),
+            nn.SiLU(),
+            zero_module(conv_nd(dims, input_ch, out_channels, 3, padding=1)),
+        )
+
+    def convert_to_fp16(self):
+        """
+        Convert the torso of the model to float16.
+        """
+        self.input_blocks.apply(convert_module_to_f16)
+        self.middle_block.apply(convert_module_to_f16)
+        self.output_blocks.apply(convert_module_to_f16)
+
+    def convert_to_fp32(self):
+        """
+        Convert the torso of the model to float32.
+        """
+        self.input_blocks.apply(convert_module_to_f32)
+        self.middle_block.apply(convert_module_to_f32)
+        self.output_blocks.apply(convert_module_to_f32)
+
+    def forward(self, x, timesteps, y=None):
+        """
+        Apply the model to an input batch.
+
+        :param x: an [N x C x ...] Tensor of inputs.
+        :param timesteps: a 1-D batch of timesteps.
+        :param y: an [N] Tensor of labels, if class-conditional.
+        :return: an [N x C x ...] Tensor of outputs.
+        """
+        assert (y is not None) == (
+            self.num_classes is not None
+        ), "must specify y if and only if the model is class-conditional"
+
+        hs = []
+        emb = self.time_embed(timestep_embedding(timesteps, self.model_channels))
+
+        if self.num_classes is not None:
+            assert y.shape == (x.shape[0],)
+            emb = emb + self.label_emb(y)
+
+        h = x.type(self.dtype)
+        for module in self.input_blocks:
+            h = module(h, emb)
+            hs.append(h)
+        h = self.middle_block(h, emb)
+        for module in self.output_blocks:
+            h = th.cat([h, hs.pop()], dim=1)
+            h = module(h, emb)
+        h = h.type(x.dtype)
+        return self.out(h)
+
+
+class SuperResModel(UNetModel):
+    """
+    A UNetModel that performs super-resolution.
+
+    Expects an extra kwarg `low_res` to condition on a low-resolution image.
+    """
+
+    def __init__(self, image_size, in_channels, *args, **kwargs):
+        super().__init__(image_size, in_channels * 2, *args, **kwargs)
+
+    def forward(self, x, timesteps, low_res=None, **kwargs):
+        _, _, new_height, new_width = x.shape
+        upsampled = F.interpolate(low_res, (new_height, new_width), mode="bilinear")
+        x = th.cat([x, upsampled], dim=1)
+        return super().forward(x, timesteps, **kwargs)
+
+
+class EncoderUNetModel(nn.Module):
+    """
+    The half UNet model with attention and timestep embedding.
+
+    For usage, see UNet.
+    """
+
+    def __init__(
+        self,
+        image_size,
+        in_channels,
+        model_channels,
+        out_channels,
+        num_res_blocks,
+        attention_resolutions,
+        dropout=0,
+        channel_mult=(1, 2, 4, 8),
+        conv_resample=True,
+        dims=2,
+        use_checkpoint=False,
+        use_fp16=False,
+        num_heads=1,
+        num_head_channels=-1,
+        num_heads_upsample=-1,
+        use_scale_shift_norm=False,
+        resblock_updown=False,
+        use_new_attention_order=False,
+        pool="adaptive",
+    ):
+        super().__init__()
+
+        if num_heads_upsample == -1:
+            num_heads_upsample = num_heads
+
+        self.in_channels = in_channels
+        self.model_channels = model_channels
+        self.out_channels = out_channels
+        self.num_res_blocks = num_res_blocks
+        self.attention_resolutions = attention_resolutions
+        self.dropout = dropout
+        self.channel_mult = channel_mult
+        self.conv_resample = conv_resample
+        self.use_checkpoint = use_checkpoint
+        self.dtype = th.float16 if use_fp16 else th.float32
+        self.num_heads = num_heads
+        self.num_head_channels = num_head_channels
+        self.num_heads_upsample = num_heads_upsample
+
+        time_embed_dim = model_channels * 4
+        self.time_embed = nn.Sequential(
+            linear(model_channels, time_embed_dim),
+            nn.SiLU(),
+            linear(time_embed_dim, time_embed_dim),
+        )
+
+        ch = int(channel_mult[0] * model_channels)
+        self.input_blocks = nn.ModuleList(
+            [TimestepEmbedSequential(conv_nd(dims, in_channels, ch, 3, padding=1))]
+        )
+        self._feature_size = ch
+        input_block_chans = [ch]
+        ds = 1
+        for level, mult in enumerate(channel_mult):
+            for _ in range(num_res_blocks):
+                layers = [
+                    ResBlock(
+                        ch,
+                        time_embed_dim,
+                        dropout,
+                        out_channels=int(mult * model_channels),
+                        dims=dims,
+                        use_checkpoint=use_checkpoint,
+                        use_scale_shift_norm=use_scale_shift_norm,
+                    )
+                ]
+                ch = int(mult * model_channels)
+                if ds in attention_resolutions:
+                    layers.append(
+                        AttentionBlock(
+                            ch,
+                            use_checkpoint=use_checkpoint,
+                            num_heads=num_heads,
+                            num_head_channels=num_head_channels,
+                            use_new_attention_order=use_new_attention_order,
+                        )
+                    )
+                self.input_blocks.append(TimestepEmbedSequential(*layers))
+                self._feature_size += ch
+                input_block_chans.append(ch)
+            if level != len(channel_mult) - 1:
+                out_ch = ch
+                self.input_blocks.append(
+                    TimestepEmbedSequential(
+                        ResBlock(
+                            ch,
+                            time_embed_dim,
+                            dropout,
+                            out_channels=out_ch,
+                            dims=dims,
+                            use_checkpoint=use_checkpoint,
+                            use_scale_shift_norm=use_scale_shift_norm,
+                            down=True,
+                        )
+                        if resblock_updown
+                        else Downsample(
+                            ch, conv_resample, dims=dims, out_channels=out_ch
+                        )
+                    )
+                )
+                ch = out_ch
+                input_block_chans.append(ch)
+                ds *= 2
+                self._feature_size += ch
+
+        self.middle_block = TimestepEmbedSequential(
+            ResBlock(
+                ch,
+                time_embed_dim,
+                dropout,
+                dims=dims,
+                use_checkpoint=use_checkpoint,
+                use_scale_shift_norm=use_scale_shift_norm,
+            ),
+            AttentionBlock(
+                ch,
+                use_checkpoint=use_checkpoint,
+                num_heads=num_heads,
+                num_head_channels=num_head_channels,
+                use_new_attention_order=use_new_attention_order,
+            ),
+            ResBlock(
+                ch,
+                time_embed_dim,
+                dropout,
+                dims=dims,
+                use_checkpoint=use_checkpoint,
+                use_scale_shift_norm=use_scale_shift_norm,
+            ),
+        )
+        self._feature_size += ch
+        self.pool = pool
+        if pool == "adaptive":
+            self.out = nn.Sequential(
+                normalization(ch),
+                nn.SiLU(),
+                nn.AdaptiveAvgPool2d((1, 1)),
+                zero_module(conv_nd(dims, ch, out_channels, 1)),
+                nn.Flatten(),
+            )
+        elif pool == "attention":
+            assert num_head_channels != -1
+            self.out = nn.Sequential(
+                normalization(ch),
+                nn.SiLU(),
+                AttentionPool2d(
+                    (image_size // ds), ch, num_head_channels, out_channels
+                ),
+            )
+        elif pool == "spatial":
+            self.out = nn.Sequential(
+                nn.Linear(self._feature_size, 2048),
+                nn.ReLU(),
+                nn.Linear(2048, self.out_channels),
+            )
+        elif pool == "spatial_v2":
+            self.out = nn.Sequential(
+                nn.Linear(self._feature_size, 2048),
+                normalization(2048),
+                nn.SiLU(),
+                nn.Linear(2048, self.out_channels),
+            )
+        else:
+            raise NotImplementedError(f"Unexpected {pool} pooling")
+
+    def convert_to_fp16(self):
+        """
+        Convert the torso of the model to float16.
+        """
+        self.input_blocks.apply(convert_module_to_f16)
+        self.middle_block.apply(convert_module_to_f16)
+
+    def convert_to_fp32(self):
+        """
+        Convert the torso of the model to float32.
+        """
+        self.input_blocks.apply(convert_module_to_f32)
+        self.middle_block.apply(convert_module_to_f32)
+
+    def forward(self, x, timesteps):
+        """
+        Apply the model to an input batch.
+
+        :param x: an [N x C x ...] Tensor of inputs.
+        :param timesteps: a 1-D batch of timesteps.
+        :return: an [N x K] Tensor of outputs.
+        """
+        emb = self.time_embed(timestep_embedding(timesteps, self.model_channels))
+
+        results = []
+        h = x.type(self.dtype)
+        for module in self.input_blocks:
+            h = module(h, emb)
+            if self.pool.startswith("spatial"):
+                results.append(h.type(x.dtype).mean(dim=(2, 3)))
+        h = self.middle_block(h, emb)
+        if self.pool.startswith("spatial"):
+            results.append(h.type(x.dtype).mean(dim=(2, 3)))
+            h = th.cat(results, axis=-1)
+            return self.out(h)
+        else:
+            h = h.type(x.dtype)
+            return self.out(h)

highres_final_vis.py

@@ -0,0 +1,124 @@
+import numpy as np
+import torch
+from einops import rearrange
+
+from voxnerf.render import subpixel_rays_from_img
+
+from run_sjc import (
+    SJC, ScoreAdapter, StableDiffusion,
+    tqdm, EventStorage, HeartBeat, EarlyLoopBreak, get_event_storage, get_heartbeat, optional_load_config, read_stats,
+    vis_routine, stitch_vis, latest_ckpt,
+    scene_box_filter, render_ray_bundle, as_torch_tsrs,
+    device_glb
+)
+
+
+# the SD deocder is very memory hungry; the latent image cannot be too large
+# for a graphics card with < 12 GB memory, set this to 128; quality already good
+# if your card has 12 to 24 GB memory, you can set this to 200;
+# but visually it won't help beyond a certain point. Our teaser is done with 128.
+decoder_bottleneck_hw = 128
+
+
+def final_vis():
+    cfg = optional_load_config(fname="full_config.yml")
+    assert len(cfg) > 0, "can't find cfg file"
+    mod = SJC(**cfg)
+
+    family = cfg.pop("family")
+    model: ScoreAdapter = getattr(mod, family).make()
+    vox = mod.vox.make()
+    poser = mod.pose.make()
+
+    pbar = tqdm(range(1))
+
+    with EventStorage(), HeartBeat(pbar):
+        ckpt_fname = latest_ckpt()
+        state = torch.load(ckpt_fname, map_location="cpu")
+        vox.load_state_dict(state)
+        vox.to(device_glb)
+
+        with EventStorage("highres"):
+            # what dominates the speed is NOT the factor here.
+            # you can try from 2 to 8, and the speed is about the same.
+            # the dominating factor in the pipeline I believe is the SD decoder.
+            evaluate(model, vox, poser, n_frames=200, factor=4)
+
+
+@torch.no_grad()
+def evaluate(score_model, vox, poser, n_frames=200, factor=4):
+    H, W = poser.H, poser.W
+    vox.eval()
+    K, poses = poser.sample_test(n_frames)
+    del n_frames
+    poses = poses[60:]  # skip the full overhead view; not interesting
+
+    fuse = EarlyLoopBreak(5)
+    metric = get_event_storage()
+    hbeat = get_heartbeat()
+
+    aabb = vox.aabb.T.cpu().numpy()
+    vox = vox.to(device_glb)
+
+    num_imgs = len(poses)
+
+    for i in (pbar := tqdm(range(num_imgs))):
+        if fuse.on_break():
+            break
+
+        pose = poses[i]
+        y, depth = highres_render_one_view(vox, aabb, H, W, K, pose, f=factor)
+        if isinstance(score_model, StableDiffusion):
+            y = score_model.decode(y)
+        vis_routine(metric, y, depth)
+
+        metric.step()
+        hbeat.beat()
+
+    metric.flush_history()
+
+    metric.put_artifact(
+        "movie_im_and_depth", ".mp4",
+        lambda fn: stitch_vis(fn, read_stats(metric.output_dir, "view")[1])
+    )
+
+    metric.put_artifact(
+        "movie_im_only", ".mp4",
+        lambda fn: stitch_vis(fn, read_stats(metric.output_dir, "img")[1])
+    )
+
+    metric.step()
+
+
+def highres_render_one_view(vox, aabb, H, W, K, pose, f=4):
+    bs = 4096
+
+    ro, rd = subpixel_rays_from_img(H, W, K, pose, f=f)
+    ro, rd, t_min, t_max = scene_box_filter(ro, rd, aabb)
+    n = len(ro)
+    ro, rd, t_min, t_max = as_torch_tsrs(vox.device, ro, rd, t_min, t_max)
+
+    rgbs = torch.zeros(n, 4, device=vox.device)
+    depth = torch.zeros(n, 1, device=vox.device)
+
+    with torch.no_grad():
+        for i in range(int(np.ceil(n / bs))):
+            s = i * bs
+            e = min(n, s + bs)
+            _rgbs, _depth, _ = render_ray_bundle(
+                vox, ro[s:e], rd[s:e], t_min[s:e], t_max[s:e]
+            )
+            rgbs[s:e] = _rgbs
+            depth[s:e] = _depth
+
+    rgbs = rearrange(rgbs, "(h w) c -> 1 c h w", h=H*f, w=W*f)
+    depth = rearrange(depth, "(h w) 1 -> h w", h=H*f, w=W*f)
+    rgbs = torch.nn.functional.interpolate(
+        rgbs, (decoder_bottleneck_hw, decoder_bottleneck_hw),
+        mode='bilinear', antialias=True
+    )
+    return rgbs, depth
+
+
+if __name__ == "__main__":
+    final_vis()

misc.py

@@ -0,0 +1,53 @@
+import numpy as np
+import torch
+
+
+def torch_samps_to_imgs(imgs, uncenter=True):
+    if uncenter:
+        imgs = (imgs + 1) / 2  # [-1, 1] -> [0, 1]
+    imgs = (imgs * 255).clamp(0, 255)
+    imgs = imgs.to(torch.uint8)
+    imgs = imgs.permute(0, 2, 3, 1)
+    imgs = imgs.cpu().numpy()
+    return imgs
+
+
+def imgs_to_torch(imgs):
+    assert imgs.dtype == np.uint8
+    assert len(imgs.shape) == 4 and imgs.shape[-1] == 3, "expect (N, H, W, C)"
+    _, H, W, _ = imgs.shape
+
+    imgs = imgs.transpose(0, 3, 1, 2)
+    imgs = (imgs / 255).astype(np.float32)
+    imgs = (imgs * 2) - 1
+    imgs = torch.as_tensor(imgs)
+    H, W = [_l - (_l % 32) for _l in (H, W)]
+    imgs = torch.nn.functional.interpolate(imgs, (H, W), mode="bilinear")
+    return imgs
+
+
+def test_encode_decode():
+    import imageio
+    from run_img_sampling import ScoreAdapter, SD
+    from vis import _draw
+
+    fname = "~/clean.png"
+    raw = imageio.imread(fname)
+    raw = imgs_to_torch(raw[np.newaxis, ...])
+
+    model: ScoreAdapter = SD().run()
+    raw = raw.to(model.device)
+    zs = model.encode(raw)
+    img = model.decode(zs)
+    img = torch_samps_to_imgs(img)
+    _draw(
+        [imageio.imread(fname), img.squeeze(0)],
+    )
+
+
+def test():
+    test_encode_decode()
+
+
+if __name__ == "__main__":
+    test()

my/README.md

@@ -0,0 +1,2 @@
+a personal tookit for experiment management;
+some of the designs patterns are inspired by detectron2

my/config.py

@@ -0,0 +1,234 @@
+from typing import List, Union
+from copy import deepcopy
+from collections import namedtuple
+from pathlib import Path
+import argparse
+from argparse import RawDescriptionHelpFormatter
+import yaml
+from pydantic import BaseModel as _Base
+
+
+class BaseConf(_Base):
+    class Config:
+        validate_all = True
+        allow_mutation = True
+        extra = "ignore"
+
+
+def SingleOrList(inner_type):
+    return Union[inner_type, List[inner_type]]
+
+
+def optional_load_config(fname="config.yml"):
+    cfg = {}
+    conf_fname = Path.cwd() / fname
+    if conf_fname.is_file():
+        with conf_fname.open("r") as f:
+            raw = f.read()
+            print("loaded config\n ")
+            print(raw)  # yaml raw itself is well formatted
+            cfg = yaml.safe_load(raw)
+    return cfg
+
+
+def write_full_config(cfg_obj, fname="full_config.yml"):
+    cfg = cfg_obj.dict()
+    cfg = _dict_to_yaml(cfg)
+    print(f"\n--- full config ---\n\n{cfg}\n")
+    with (Path.cwd() / fname).open("w") as f:
+        f.write(cfg)
+
+
+def argparse_cfg_template(curr_cfgs):
+    parser = argparse.ArgumentParser(
+        description='Manual spec of configs',
+        epilog=f'curr cfgs:\n\n{_dict_to_yaml(curr_cfgs)}',
+        formatter_class=RawDescriptionHelpFormatter
+    )
+    _, args = parser.parse_known_args()
+    clauses = []
+    for i in range(0, len(args), 2):
+        assert args[i][:2] == "--", "please start args with --"
+        clauses.append({args[i][2:]: args[i+1]})
+    print(f"cmdline clauses: {clauses}")
+
+    maker = ConfigMaker(curr_cfgs)
+    for clu in clauses:
+        maker.execute_clause(clu)
+
+    final = maker.state.copy()
+    return final
+
+
+def _dict_to_yaml(arg):
+    return yaml.safe_dump(arg, sort_keys=False, allow_unicode=True)
+
+
+def dispatch(module):
+    cfg = optional_load_config()
+    cfg = module(**cfg).dict()
+
+    cfg = argparse_cfg_template(cfg)  # cmdline takes priority
+    mod = module(**cfg)
+
+    write_full_config(mod)
+
+    mod.run()
+
+
+# below are some support tools
+
+
+class ConfigMaker():
+    CMD = namedtuple('cmd', field_names=['sub', 'verb', 'objs'])
+    VERBS = ('add', 'replace', 'del')
+
+    def __init__(self, base_node):
+        self.state = base_node
+        self.clauses = []
+
+    def clone(self):
+        return deepcopy(self)
+
+    def execute_clause(self, raw_clause):
+        cls = self.__class__
+        assert isinstance(raw_clause, (str, dict))
+        if isinstance(raw_clause, dict):
+            assert len(raw_clause) == 1, \
+                "a clause can only have 1 statement: {} clauses in {}".format(
+                    len(raw_clause), raw_clause
+            )
+            cmd = list(raw_clause.keys())[0]
+            arg = raw_clause[cmd]
+        else:
+            cmd = raw_clause
+            arg = None
+        cmd = self.parse_clause_cmd(cmd)
+        tracer = NodeTracer(self.state)
+        tracer.advance_pointer(path=cmd.sub)
+        if cmd.verb == cls.VERBS[0]:
+            tracer.add(cmd.objs, arg)
+        elif cmd.verb == cls.VERBS[1]:
+            tracer.replace(cmd.objs, arg)
+        elif cmd.verb == cls.VERBS[2]:
+            assert isinstance(raw_clause, str)
+            tracer.delete(cmd.objs)
+        self.state = tracer.state
+
+    @classmethod
+    def parse_clause_cmd(cls, input):
+        """
+        Args:
+            input: a string to be parsed
+        1. First test whether a verb is present
+        2. If not present, then str is a single subject, and verb is replace
+           This is a syntactical sugar that makes writing config easy
+        3. If a verb is found, whatever comes before is a subject, and after the
+           objects.
+        4. Handle the edge cases properly. Below are expected parse outputs
+        input       sub     verb        obj
+        --- No verb
+        ''          ''      replace     []
+        'a.b'       'a.b'   replace     []
+        'add'       ''      add         []
+        'P Q' err: 2 subjects
+        --- Verb present
+        'T add'     'T'     add         []
+        'T del a b' 'T'     del         [a, b]
+        'P Q add a' err: 2 subjects
+        'P add del b' err: 2 verbs
+        """
+        assert isinstance(input, str)
+        input = input.split()
+        objs = []
+        sub = ''
+        verb, verb_inx = cls.scan_for_verb(input)
+        if verb is None:
+            assert len(input) <= 1, "no verb present; more than 1 subject: {}"\
+                .format(input)
+            sub = input[0] if len(input) == 1 else ''
+            verb = cls.VERBS[1]
+        else:
+            assert not verb_inx > 1, 'verb {} at inx {}; more than 1 subject in: {}'\
+                .format(verb, verb_inx, input)
+            sub = input[0] if verb_inx == 1 else ''
+            objs = input[verb_inx + 1:]
+        cmd = cls.CMD(sub=sub, verb=verb, objs=objs)
+        return cmd
+
+    @classmethod
+    def scan_for_verb(cls, input_list):
+        assert isinstance(input_list, list)
+        counts = [ input_list.count(v) for v in cls.VERBS ]
+        presence = [ cnt > 0 for cnt in counts ]
+        if sum(presence) == 0:
+            return None, -1
+        elif sum(presence) > 1:
+            raise ValueError("multiple verbs discovered in {}".format(input_list))
+
+        if max(counts) > 1:
+            raise ValueError("verbs repeated in cmd: {}".format(input_list))
+        # by now, there is 1 verb that has occured exactly 1 time
+        verb = cls.VERBS[presence.index(1)]
+        inx = input_list.index(verb)
+        return verb, inx
+
+
+class NodeTracer():
+    def __init__(self, src_node):
+        """
+        A src node can be either a list or dict
+        """
+        assert isinstance(src_node, (list, dict))
+
+        # these are movable pointers
+        self.child_token = "_"  # init token can be anything
+        self.parent = {self.child_token: src_node}
+
+        # these are permanent pointers at the root
+        self.root_child_token = self.child_token
+        self.root = self.parent
+
+    @property
+    def state(self):
+        return self.root[self.root_child_token]
+
+    @property
+    def pointed(self):
+        return self.parent[self.child_token]
+
+    def advance_pointer(self, path):
+        if len(path) == 0:
+            return
+        path_list = list(
+            map(lambda x: int(x) if str.isdigit(x) else x, path.split('.'))
+        )
+
+        for i, token in enumerate(path_list):
+            self.parent = self.pointed
+            self.child_token = token
+            try:
+                self.pointed
+            except (IndexError, KeyError):
+                raise ValueError(
+                    "During the tracing of {}, {}-th token '{}'"
+                    " is not present in node {}".format(
+                        path, i, self.child_token, self.state
+                    )
+                )
+
+    def replace(self, objs, arg):
+        assert len(objs) == 0
+        val_type = type(self.parent[self.child_token])
+        # this is such an unfortunate hack
+        # turn everything to string, so that eval could work
+        # some of the clauses come from cmdline, some from yaml files for sow.
+        arg = str(arg)
+        if val_type == str:
+            pass
+        else:
+            arg = eval(arg)
+            assert type(arg) == val_type, \
+                f"require {val_type.__name__}, given {type(arg).__name__}"
+
+        self.parent[self.child_token] = arg

my/registry.py

@@ -0,0 +1,62 @@
+# from detectron2
+from typing import Any, Dict, Iterable, Iterator, Tuple
+from tabulate import tabulate
+
+
+class Registry(Iterable[Tuple[str, Any]]):
+    def __init__(self, name: str) -> None:
+        """
+        Args:
+            name (str): the name of this registry
+        """
+        self._name: str = name
+        self._obj_map: Dict[str, Any] = {}
+
+    def _do_register(self, name: str, obj: Any) -> None:
+        assert (
+            name not in self._obj_map
+        ), "An object named '{}' was already registered in '{}' registry!".format(
+            name, self._name
+        )
+        self._obj_map[name] = obj
+
+    def register(self, obj: Any = None) -> Any:
+        """
+        Register the given object under the the name `obj.__name__`.
+        Can be used as either a decorator or not. See docstring of this class for usage.
+        """
+        if obj is None:
+            # used as a decorator
+            def deco(func_or_class: Any) -> Any:
+                name = func_or_class.__name__
+                self._do_register(name, func_or_class)
+                return func_or_class
+
+            return deco
+
+        # used as a function call
+        name = obj.__name__
+        self._do_register(name, obj)
+
+    def get(self, name: str) -> Any:
+        ret = self._obj_map.get(name)
+        if ret is None:
+            raise KeyError(
+                "No object named '{}' found in '{}' registry!".format(name, self._name)
+            )
+        return ret
+
+    def __contains__(self, name: str) -> bool:
+        return name in self._obj_map
+
+    def __repr__(self) -> str:
+        table_headers = ["Names", "Objects"]
+        table = tabulate(
+            self._obj_map.items(), headers=table_headers, tablefmt="fancy_grid"
+        )
+        return "Registry of {}:\n".format(self._name) + table
+
+    def __iter__(self) -> Iterator[Tuple[str, Any]]:
+        return iter(self._obj_map.items())
+
+    __str__ = __repr__

my/utils/__init__.py

@@ -0,0 +1,4 @@
+from .event import EventStorage, get_event_storage, read_stats
+from .tqdm import tqdm
+from .heartbeat import HeartBeat, get_heartbeat
+from .debug import EarlyLoopBreak

my/utils/debug.py

@@ -0,0 +1,15 @@
+import os
+
+class EarlyLoopBreak():
+    def __init__(self, break_at: int):
+        self.iter = 0
+        self.break_at = break_at
+        self.on = bool(os.environ.get("EBREAK"))
+
+    def on_break(self):
+        if not self.on:
+            return
+
+        self.iter += 1
+        if self.break_at > 0 and self.iter >= self.break_at:
+            return True

my/utils/event.py

@@ -0,0 +1,142 @@
+# design inspiration from detectron2
+from pathlib import Path
+import json
+import os
+from contextlib import contextmanager
+from .ticker import IntervalTicker
+
+
+_CURRENT_STORAGE_STACK = []
+
+
+def get_event_storage():
+    """
+    Returns:
+        The :class:`EventStorage` object that's currently being used.
+        Throws an error if no :class:`EventStorage` is currently enabled.
+    """
+    assert len(
+        _CURRENT_STORAGE_STACK
+    ), "get_event_storage() has to be called inside a 'with EventStorage(...)' context!"
+    return _CURRENT_STORAGE_STACK[-1]
+
+
+def read_lined_json(fname):
+    with Path(fname).open('r') as f:
+        for line in f:
+            item = json.loads(line)
+            yield item
+
+
+def read_stats(dirname, key):
+    if dirname is None or not (fname := Path(dirname) / "history.json").is_file():
+        return [], []
+    stats = read_lined_json(fname)
+    stats = list(filter(lambda x: key in x, stats))
+    xs = [e['iter'] for e in stats]
+    ys = [e[key] for e in stats]
+    return xs, ys
+
+
+class EventStorage():
+    def __init__(self, output_dir="./", start_iter=0, flush_period=60):
+        self.iter = start_iter
+        self.ticker = IntervalTicker(flush_period)
+        self.history = []
+        self._current_prefix = ""
+        self._init_curr_buffer_()
+
+        self.output_dir = output_dir
+        self.writable = False
+
+    def _open(self):
+        if self.writable:
+            output_dir = Path(self.output_dir)
+            if not output_dir.is_dir():
+                output_dir.mkdir(parents=True, exist_ok=True)
+            json_fname = output_dir / 'history.json'
+
+            self._file_handle = json_fname.open('a', encoding='utf8')
+            self.output_dir = output_dir  # make sure it's a path object
+
+    def _init_curr_buffer_(self):
+        self.curr_buffer = {'iter': self.iter}
+
+    def step(self, flush=False):
+        self.history.append(self.curr_buffer)
+
+        on_flush_period = self.ticker.tick()
+        if flush or on_flush_period:
+            self.flush_history()
+
+        self.iter += 1
+        self._init_curr_buffer_()
+
+    def flush_history(self):
+        if self.writable:
+            for item in self.history:
+                line = json.dumps(item, sort_keys=True, ensure_ascii=False) + "\n"
+                self._file_handle.write(line)
+            self._file_handle.flush()
+        self.history = []
+
+    def full_key(self, key):
+        assert isinstance(key, str)
+        name = self._current_prefix + key
+        return name
+
+    def put(self, key, val):
+        key = self.full_key(key)
+        assert isinstance(val, (int, float, str))
+        if isinstance(val, float):
+            val = round(val, 3)
+        self.curr_buffer[key] = val
+
+    def put_scalars(self, **kwargs):
+        for k, v in kwargs.items():
+            self.put(k, v)
+
+    def put_artifact(self, key, ext, save_func):
+        if not self.writable:
+            return
+        os.makedirs(self.output_dir / key, exist_ok=True)
+        fname = (self.output_dir / key / f"step_{self.iter}").with_suffix(ext)
+        fname = str(fname)
+
+        # must be called inside so that
+        # 1. the func is not executed if the metric is not writable
+        # 2. the key is only inserted if the func succeeds
+        save_func(fname)
+        self.put(key, fname)
+        return fname
+
+    def close(self):
+        self.flush_history()
+        if self.writable:
+            self._file_handle.close()
+
+    def get_last(self):
+        if len(self.history) > 0:
+            last = self.history[-1]
+            return last
+
+    def __enter__(self):
+        if len(_CURRENT_STORAGE_STACK) > 0:
+            parent = _CURRENT_STORAGE_STACK[-1]
+            root, dirname = parent.output_dir, self.output_dir
+            if root is not None and dirname is not None:
+                child_dir = parent.output_dir / f"{self.output_dir}_{parent.iter}"
+                self.output_dir = child_dir
+                parent.put(str(dirname), str(child_dir))
+
+        if self.output_dir is not None:
+            self.writable = True
+            self._open()
+
+        _CURRENT_STORAGE_STACK.append(self)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        assert _CURRENT_STORAGE_STACK[-1] == self
+        _CURRENT_STORAGE_STACK.pop()
+        self.close()

my/utils/heartbeat.py

@@ -0,0 +1,78 @@
+# generates periodic hearbeats for remote expriment monitoring
+from pathlib import Path
+import json
+from inspect import stack
+from .ticker import IntervalTicker
+
+_CURRENT_BEAT_STACK = []
+
+
+def get_heartbeat():
+    """
+    Returns:
+        The :class:`HeartBeat` object that's currently being used.
+        Throws an error if no :class:`EventStorage` is currently enabled.
+    """
+    assert len(
+        _CURRENT_BEAT_STACK
+    ), "get_heartbeat() has to be called inside a 'with EventStorage(...)' context!"
+    return _CURRENT_BEAT_STACK[-1]
+
+
+def get_tqdm_meter(pbar, format_dict):
+    format_dict['bar_format'] = "{r_bar}"
+    meter_str = pbar.format_meter(**format_dict)
+    meter_str = meter_str[2:]
+    return meter_str
+
+
+def caller_info(n_stack_up):
+    info = stack()[1 + n_stack_up]  # 1 up as base so that it starts from caller
+    msg = f"{info.filename}:{info.lineno} - {info.function}"
+    return msg
+
+
+class HeartBeat():
+    def __init__(
+        self, pbar, write_interval=10,
+        output_dir="./", fname="heartbeat.json"
+    ):
+        self.pbar = pbar
+        self.fname = Path(output_dir) / fname
+        self.ticker = IntervalTicker(write_interval)
+        self.completed = False
+
+        # force one write at the beginning
+        self.beat(force_write=True, n_stack_up=2)
+
+    def beat(self, force_write=False, n_stack_up=1):
+        on_write_period = self.ticker.tick()
+        if force_write or on_write_period:
+            stats = self.stats()
+            stats['caller'] = caller_info(n_stack_up)
+
+            with open(self.fname, "w") as f:
+                json.dump(stats, f)
+
+    def done(self):
+        self.completed = True
+        self.beat(force_write=True, n_stack_up=2)
+
+    def stats(self):
+        pbar = self.pbar
+        fdict = pbar.format_dict
+        stats = {
+            "beat": self.ticker.tick_str(),
+            "done": self.completed,
+            "meter": get_tqdm_meter(pbar, fdict),
+            "elapsed": int(fdict['elapsed'])
+        }
+        return stats
+
+    def __enter__(self):
+        _CURRENT_BEAT_STACK.append(self)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        assert _CURRENT_BEAT_STACK[-1] == self
+        _CURRENT_BEAT_STACK.pop()

my/utils/plot.py

@@ -0,0 +1,9 @@
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+def mpl_fig_to_buffer(fig):
+    fig.canvas.draw()
+    plot = np.array(fig.canvas.renderer.buffer_rgba())
+    plt.close(fig)
+    return plot

my/utils/seed.py

@@ -0,0 +1,21 @@
+# from pytorch lightning
+import random
+import numpy as np
+import torch
+
+max_seed_value = np.iinfo(np.uint32).max
+min_seed_value = np.iinfo(np.uint32).min
+
+
+def seed_everything(seed=None):
+    seed = int(seed)
+
+    if not (min_seed_value <= seed <= max_seed_value):
+        raise ValueError(f"{seed} is not in bounds, numpy accepts from {min_seed_value} to {max_seed_value}")
+
+    print(f"seed set to {seed}")
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    return seed

my/utils/ticker.py

@@ -0,0 +1,18 @@
+from datetime import date, time, datetime, timedelta
+from time import sleep
+
+
+class IntervalTicker():
+    def __init__(self, interval=60):
+        self.interval = timedelta(seconds=interval)
+        self.last_tick = datetime.now()
+        self.now = self.last_tick
+
+    def tick(self):
+        self.now = datetime.now()
+        if (self.now - self.last_tick) > self.interval:
+            self.last_tick = self.now
+            return True
+
+    def tick_str(self):
+        return self.now.isoformat(timespec='seconds')

my/utils/tqdm.py

@@ -0,0 +1,10 @@
+import os
+from tqdm import tqdm as orig_tqdm
+
+
+def tqdm(*args, **kwargs):
+    is_remote = bool(os.environ.get("IS_REMOTE", False))
+    if is_remote:
+        f = open(os.devnull, "w")
+        kwargs.update({"file": f})
+    return orig_tqdm(*args, **kwargs)

my3d.py

@@ -0,0 +1,160 @@
+# some tools developed for the vision class
+import numpy as np
+from numpy import cross, tan
+from numpy.linalg import norm, inv
+
+
+def normalize(v):
+    return v / norm(v)
+
+
+def camera_pose(eye, front, up):
+    z = normalize(-1 * front)
+    x = normalize(cross(up, z))
+    y = normalize(cross(z, x))
+
+    # convert to col vector
+    x = x.reshape(-1, 1)
+    y = y.reshape(-1, 1)
+    z = z.reshape(-1, 1)
+    eye = eye.reshape(-1, 1)
+
+    pose = np.block([
+        [x, y, z, eye],
+        [0, 0, 0, 1]
+    ])
+    return pose
+
+
+def compute_extrinsics(eye, front, up):
+    pose = camera_pose(eye, front, up)
+    world_2_cam = inv(pose)
+    return world_2_cam
+
+
+def compute_intrinsics(aspect_ratio, fov, img_height_in_pix):
+    # aspect ratio is  w / h
+    ndc = compute_proj_to_normalized(aspect_ratio, fov)
+
+    # anything beyond [-1, 1] should be discarded
+    # this did not mention how to do z-clipping;
+
+    ndc_to_img = compute_normalized_to_img_trans(aspect_ratio, img_height_in_pix)
+    intrinsic = ndc_to_img @ ndc
+    return intrinsic
+
+
+def compute_proj_to_normalized(aspect, fov):
+    # compared to standard OpenGL NDC intrinsic,
+    # this skips the 3rd row treatment on z. hence the name partial_ndc
+    fov_in_rad = fov / 180 * np.pi
+    t = tan(fov_in_rad / 2)  # tan half fov
+    partial_ndc_intrinsic = np.array([
+        [1 / (t * aspect), 0, 0, 0],
+        [0, 1 / t, 0, 0],
+        [0, 0, -1, 0]  # copy the negative distance for division
+    ])
+    return partial_ndc_intrinsic
+
+
+def compute_normalized_to_img_trans(aspect, img_height_in_pix):
+    img_h = img_height_in_pix
+    img_w = img_height_in_pix * aspect
+
+    # note the OpenGL convention that (0, 0) sits at the center of the pixel;
+    # hence the extra -0.5 translation
+    # this is useful when you shoot rays through a pixel to the scene
+    ndc_to_img = np.array([
+        [img_w / 2, 0, img_w / 2 - 0.5],
+        [0, img_h / 2, img_h / 2 - 0.5],
+        [0, 0, 1]
+    ])
+
+    img_y_coord_flip = np.array([
+        [1, 0, 0],
+        [0, -1, img_h - 1],  # note the -1
+        [0, 0, 1]
+    ])
+
+    # the product of the above 2 matrices is equivalent to adding
+    # - sign to the (1, 1) entry
+    # you could have simply written
+    # ndc_to_img = np.array([
+    #     [img_w / 2, 0, img_w / 2 - 0.5],
+    #     [0, -img_h / 2, img_h / 2 - 0.5],
+    #     [0, 0, 1]
+    # ])
+
+    ndc_to_img = img_y_coord_flip @ ndc_to_img
+    return ndc_to_img
+
+
+def unproject(K, pixel_coords, depth=1.0):
+    """sometimes also referred to as backproject
+        pixel_coords: [n, 2] pixel locations
+        depth: [n,] or [,] depth value. of a shape that is broadcastable with pix coords
+    """
+    K = K[0:3, 0:3]
+
+    pixel_coords = as_homogeneous(pixel_coords)
+    pixel_coords = pixel_coords.T  # [2+1, n], so that mat mult is on the left
+
+    # this will give points with z = -1, which is exactly what you want since
+    # your camera is facing the -ve z axis
+    pts = inv(K) @ pixel_coords
+
+    pts = pts * depth  # [3, n] * [n,] broadcast
+    pts = pts.T
+    pts = as_homogeneous(pts)
+    return pts
+
+
+"""
+these two functions are changed so that they can handle arbitrary number of
+dimensions >=1
+"""
+
+
+def homogenize(pts):
+    # pts: [..., d], where last dim of the d is the diviser
+    *front, d = pts.shape
+    pts = pts / pts[..., -1].reshape(*front, 1)
+    return pts
+
+
+def as_homogeneous(pts, lib=np):
+    # pts: [..., d]
+    *front, d = pts.shape
+    points = lib.ones((*front, d + 1))
+    points[..., :d] = pts
+    return points
+
+
+def simple_point_render(pts, img_w, img_h, fov, eye, front, up):
+    """
+    pts: [N, 3]
+    """
+    canvas = np.ones((img_h, img_w, 3))
+
+    pts = as_homogeneous(pts)
+
+    E = compute_extrinsics(eye, front, up)
+    world_2_ndc = compute_proj_to_normalized(img_w / img_h, fov)
+    ndc_to_img = compute_normalized_to_img_trans(img_w / img_h, img_h)
+
+    pts = pts @ E.T
+    pts = pts @ world_2_ndc.T
+    pts = homogenize(pts)
+
+    # now filter out outliers beyond [-1, 1]
+    outlier_mask = (np.abs(pts) > 1.0).any(axis=1)
+    pts = pts[~outlier_mask]
+
+    pts = pts @ ndc_to_img.T
+
+    # now draw each point
+    pts = np.rint(pts).astype(np.int32)
+    xs, ys, _ = pts.T
+    canvas[ys, xs] = (1, 0, 0)
+
+    return canvas

ncsn/bedroom.yml

@@ -0,0 +1,69 @@
+training:
+  batch_size: 128
+  n_epochs: 500000
+  n_iters: 150001
+  snapshot_freq: 5000
+  snapshot_sampling: true
+  anneal_power: 2
+  log_all_sigmas: false
+
+sampling:
+  batch_size: 36
+  data_init: false
+  step_lr: 0.0000018
+  n_steps_each: 3
+  ckpt_id: 150000
+  final_only: true
+  fid: false
+  denoise: true
+  num_samples4fid: 10000
+  inpainting: false
+  interpolation: false
+  n_interpolations: 10
+
+fast_fid:
+  batch_size: 1000
+  num_samples: 1000
+  step_lr: 0.0000018
+  n_steps_each: 3
+  begin_ckpt: 100000
+  end_ckpt: 150000
+  verbose: false
+  ensemble: false
+
+test:
+  begin_ckpt: 5000
+  end_ckpt: 150000
+  batch_size: 100
+
+data:
+  dataset: "LSUN"
+  category: "bedroom"
+  image_size: 128
+  channels: 3
+  logit_transform: false
+  uniform_dequantization: false
+  gaussian_dequantization: false
+  random_flip: true
+  rescaled: false
+  num_workers: 32
+
+model:
+  sigma_begin: 190
+  num_classes: 1086
+  ema: true
+  ema_rate: 0.999
+  spec_norm: false
+  sigma_dist: geometric
+  sigma_end: 0.01
+  normalization: InstanceNorm++
+  nonlinearity: elu
+  ngf: 128
+
+optim:
+  weight_decay: 0.000
+  optimizer: "Adam"
+  lr: 0.0001
+  beta1: 0.9
+  amsgrad: false
+  eps: 0.00000001

ncsn/ema.py

@@ -0,0 +1,47 @@
+import copy
+import torch.nn as nn
+
+class EMAHelper(object):
+    def __init__(self, mu=0.999):
+        self.mu = mu
+        self.shadow = {}
+
+    def register(self, module):
+        if isinstance(module, nn.DataParallel):
+            module = module.module
+        for name, param in module.named_parameters():
+            if param.requires_grad:
+                self.shadow[name] = param.data.clone()
+
+    def update(self, module):
+        if isinstance(module, nn.DataParallel):
+            module = module.module
+        for name, param in module.named_parameters():
+            if param.requires_grad:
+                self.shadow[name].data = (1. - self.mu) * param.data + self.mu * self.shadow[name].data
+
+    def ema(self, module):
+        if isinstance(module, nn.DataParallel):
+            module = module.module
+        for name, param in module.named_parameters():
+            if param.requires_grad:
+                param.data.copy_(self.shadow[name].data)
+
+    def ema_copy(self, module):
+        if isinstance(module, nn.DataParallel):
+            inner_module = module.module
+            module_copy = type(inner_module)(inner_module.config).to(inner_module.config.device)
+            module_copy.load_state_dict(inner_module.state_dict())
+            module_copy = nn.DataParallel(module_copy)
+        else:
+            module_copy = type(module)(module.config).to(module.config.device)
+            module_copy.load_state_dict(module.state_dict())
+        # module_copy = copy.deepcopy(module)
+        self.ema(module_copy)
+        return module_copy
+
+    def state_dict(self):
+        return self.shadow
+
+    def load_state_dict(self, state_dict):
+        self.shadow = state_dict

ncsn/layers.py

@@ -0,0 +1,456 @@
+import torch.nn as nn
+import torch
+from torch.nn.parameter import Parameter
+import torch.nn.functional as F
+from .normalization import *
+from functools import partial
+import math
+import torch.nn.init as init
+
+
+def get_act(config):
+    if config.model.nonlinearity.lower() == 'elu':
+        return nn.ELU()
+    elif config.model.nonlinearity.lower() == 'relu':
+        return nn.ReLU()
+    elif config.model.nonlinearity.lower() == 'lrelu':
+        return nn.LeakyReLU(negative_slope=0.2)
+    elif config.model.nonlinearity.lower() == 'swish':
+        def swish(x):
+            return x * torch.sigmoid(x)
+        return swish
+    else:
+        raise NotImplementedError('activation function does not exist!')
+
+def spectral_norm(layer, n_iters=1):
+    return torch.nn.utils.spectral_norm(layer, n_power_iterations=n_iters)
+
+def conv1x1(in_planes, out_planes, stride=1, bias=True, spec_norm=False):
+    "1x1 convolution"
+    conv = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride,
+                     padding=0, bias=bias)
+    if spec_norm:
+        conv = spectral_norm(conv)
+    return conv
+
+
+def conv3x3(in_planes, out_planes, stride=1, bias=True, spec_norm=False):
+    "3x3 convolution with padding"
+    conv = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
+                     padding=1, bias=bias)
+    if spec_norm:
+        conv = spectral_norm(conv)
+
+    return conv
+
+
+def stride_conv3x3(in_planes, out_planes, kernel_size, bias=True, spec_norm=False):
+    conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=2,
+                     padding=kernel_size // 2, bias=bias)
+    if spec_norm:
+        conv = spectral_norm(conv)
+    return conv
+
+
+def dilated_conv3x3(in_planes, out_planes, dilation, bias=True, spec_norm=False):
+    conv = nn.Conv2d(in_planes, out_planes, kernel_size=3, padding=dilation, dilation=dilation, bias=bias)
+    if spec_norm:
+        conv = spectral_norm(conv)
+
+    return conv
+
+class CRPBlock(nn.Module):
+    def __init__(self, features, n_stages, act=nn.ReLU(), maxpool=True, spec_norm=False):
+        super().__init__()
+        self.convs = nn.ModuleList()
+        for i in range(n_stages):
+            self.convs.append(conv3x3(features, features, stride=1, bias=False, spec_norm=spec_norm))
+        self.n_stages = n_stages
+        if maxpool:
+            self.maxpool = nn.MaxPool2d(kernel_size=5, stride=1, padding=2)
+        else:
+            self.maxpool = nn.AvgPool2d(kernel_size=5, stride=1, padding=2)
+
+        self.act = act
+
+    def forward(self, x):
+        x = self.act(x)
+        path = x
+        for i in range(self.n_stages):
+            path = self.maxpool(path)
+            path = self.convs[i](path)
+            x = path + x
+        return x
+
+
+class CondCRPBlock(nn.Module):
+    def __init__(self, features, n_stages, num_classes, normalizer, act=nn.ReLU(), spec_norm=False):
+        super().__init__()
+        self.convs = nn.ModuleList()
+        self.norms = nn.ModuleList()
+        self.normalizer = normalizer
+        for i in range(n_stages):
+            self.norms.append(normalizer(features, num_classes, bias=True))
+            self.convs.append(conv3x3(features, features, stride=1, bias=False, spec_norm=spec_norm))
+
+        self.n_stages = n_stages
+        self.maxpool = nn.AvgPool2d(kernel_size=5, stride=1, padding=2)
+        self.act = act
+
+    def forward(self, x, y):
+        x = self.act(x)
+        path = x
+        for i in range(self.n_stages):
+            path = self.norms[i](path, y)
+            path = self.maxpool(path)
+            path = self.convs[i](path)
+
+            x = path + x
+        return x
+
+
+class RCUBlock(nn.Module):
+    def __init__(self, features, n_blocks, n_stages, act=nn.ReLU(), spec_norm=False):
+        super().__init__()
+
+        for i in range(n_blocks):
+            for j in range(n_stages):
+                setattr(self, '{}_{}_conv'.format(i + 1, j + 1), conv3x3(features, features, stride=1, bias=False,
+                                                                         spec_norm=spec_norm))
+
+        self.stride = 1
+        self.n_blocks = n_blocks
+        self.n_stages = n_stages
+        self.act = act
+
+    def forward(self, x):
+        for i in range(self.n_blocks):
+            residual = x
+            for j in range(self.n_stages):
+                x = self.act(x)
+                x = getattr(self, '{}_{}_conv'.format(i + 1, j + 1))(x)
+
+            x += residual
+        return x
+
+
+class CondRCUBlock(nn.Module):
+    def __init__(self, features, n_blocks, n_stages, num_classes, normalizer, act=nn.ReLU(), spec_norm=False):
+        super().__init__()
+
+        for i in range(n_blocks):
+            for j in range(n_stages):
+                setattr(self, '{}_{}_norm'.format(i + 1, j + 1), normalizer(features, num_classes, bias=True))
+                setattr(self, '{}_{}_conv'.format(i + 1, j + 1),
+                        conv3x3(features, features, stride=1, bias=False, spec_norm=spec_norm))
+
+        self.stride = 1
+        self.n_blocks = n_blocks
+        self.n_stages = n_stages
+        self.act = act
+        self.normalizer = normalizer
+
+    def forward(self, x, y):
+        for i in range(self.n_blocks):
+            residual = x
+            for j in range(self.n_stages):
+                x = getattr(self, '{}_{}_norm'.format(i + 1, j + 1))(x, y)
+                x = self.act(x)
+                x = getattr(self, '{}_{}_conv'.format(i + 1, j + 1))(x)
+
+            x += residual
+        return x
+
+
+class MSFBlock(nn.Module):
+    def __init__(self, in_planes, features, spec_norm=False):
+        """
+        :param in_planes: tuples of input planes
+        """
+        super().__init__()
+        assert isinstance(in_planes, list) or isinstance(in_planes, tuple)
+        self.convs = nn.ModuleList()
+        self.features = features
+
+        for i in range(len(in_planes)):
+            self.convs.append(conv3x3(in_planes[i], features, stride=1, bias=True, spec_norm=spec_norm))
+
+    def forward(self, xs, shape):
+        sums = torch.zeros(xs[0].shape[0], self.features, *shape, device=xs[0].device)
+        for i in range(len(self.convs)):
+            h = self.convs[i](xs[i])
+            h = F.interpolate(h, size=shape, mode='bilinear', align_corners=True)
+            sums += h
+        return sums
+
+
+class CondMSFBlock(nn.Module):
+    def __init__(self, in_planes, features, num_classes, normalizer, spec_norm=False):
+        """
+        :param in_planes: tuples of input planes
+        """
+        super().__init__()
+        assert isinstance(in_planes, list) or isinstance(in_planes, tuple)
+
+        self.convs = nn.ModuleList()
+        self.norms = nn.ModuleList()
+        self.features = features
+        self.normalizer = normalizer
+
+        for i in range(len(in_planes)):
+            self.convs.append(conv3x3(in_planes[i], features, stride=1, bias=True, spec_norm=spec_norm))
+            self.norms.append(normalizer(in_planes[i], num_classes, bias=True))
+
+    def forward(self, xs, y, shape):
+        sums = torch.zeros(xs[0].shape[0], self.features, *shape, device=xs[0].device)
+        for i in range(len(self.convs)):
+            h = self.norms[i](xs[i], y)
+            h = self.convs[i](h)
+            h = F.interpolate(h, size=shape, mode='bilinear', align_corners=True)
+            sums += h
+        return sums
+
+
+class RefineBlock(nn.Module):
+    def __init__(self, in_planes, features, act=nn.ReLU(), start=False, end=False, maxpool=True, spec_norm=False):
+        super().__init__()
+
+        assert isinstance(in_planes, tuple) or isinstance(in_planes, list)
+        self.n_blocks = n_blocks = len(in_planes)
+
+        self.adapt_convs = nn.ModuleList()
+        for i in range(n_blocks):
+            self.adapt_convs.append(
+                RCUBlock(in_planes[i], 2, 2, act, spec_norm=spec_norm)
+            )
+
+        self.output_convs = RCUBlock(features, 3 if end else 1, 2, act, spec_norm=spec_norm)
+
+        if not start:
+            self.msf = MSFBlock(in_planes, features, spec_norm=spec_norm)
+
+        self.crp = CRPBlock(features, 2, act, maxpool=maxpool, spec_norm=spec_norm)
+
+    def forward(self, xs, output_shape):
+        assert isinstance(xs, tuple) or isinstance(xs, list)
+        hs = []
+        for i in range(len(xs)):
+            h = self.adapt_convs[i](xs[i])
+            hs.append(h)
+
+        if self.n_blocks > 1:
+            h = self.msf(hs, output_shape)
+        else:
+            h = hs[0]
+
+        h = self.crp(h)
+        h = self.output_convs(h)
+
+        return h
+
+
+
+class CondRefineBlock(nn.Module):
+    def __init__(self, in_planes, features, num_classes, normalizer, act=nn.ReLU(), start=False, end=False, spec_norm=False):
+        super().__init__()
+
+        assert isinstance(in_planes, tuple) or isinstance(in_planes, list)
+        self.n_blocks = n_blocks = len(in_planes)
+
+        self.adapt_convs = nn.ModuleList()
+        for i in range(n_blocks):
+            self.adapt_convs.append(
+                CondRCUBlock(in_planes[i], 2, 2, num_classes, normalizer, act, spec_norm=spec_norm)
+            )
+
+        self.output_convs = CondRCUBlock(features, 3 if end else 1, 2, num_classes, normalizer, act, spec_norm=spec_norm)
+
+        if not start:
+            self.msf = CondMSFBlock(in_planes, features, num_classes, normalizer, spec_norm=spec_norm)
+
+        self.crp = CondCRPBlock(features, 2, num_classes, normalizer, act, spec_norm=spec_norm)
+
+    def forward(self, xs, y, output_shape):
+        assert isinstance(xs, tuple) or isinstance(xs, list)
+        hs = []
+        for i in range(len(xs)):
+            h = self.adapt_convs[i](xs[i], y)
+            hs.append(h)
+
+        if self.n_blocks > 1:
+            h = self.msf(hs, y, output_shape)
+        else:
+            h = hs[0]
+
+        h = self.crp(h, y)
+        h = self.output_convs(h, y)
+
+        return h
+
+
+class ConvMeanPool(nn.Module):
+    def __init__(self, input_dim, output_dim, kernel_size=3, biases=True, adjust_padding=False, spec_norm=False):
+        super().__init__()
+        if not adjust_padding:
+            conv = nn.Conv2d(input_dim, output_dim, kernel_size, stride=1, padding=kernel_size // 2, bias=biases)
+            if spec_norm:
+                conv = spectral_norm(conv)
+            self.conv = conv
+        else:
+            conv = nn.Conv2d(input_dim, output_dim, kernel_size, stride=1, padding=kernel_size // 2, bias=biases)
+            if spec_norm:
+                conv = spectral_norm(conv)
+
+            self.conv = nn.Sequential(
+                nn.ZeroPad2d((1, 0, 1, 0)),
+                conv
+            )
+
+    def forward(self, inputs):
+        output = self.conv(inputs)
+        output = sum([output[:, :, ::2, ::2], output[:, :, 1::2, ::2],
+                      output[:, :, ::2, 1::2], output[:, :, 1::2, 1::2]]) / 4.
+        return output
+
+class MeanPoolConv(nn.Module):
+    def __init__(self, input_dim, output_dim, kernel_size=3, biases=True, spec_norm=False):
+        super().__init__()
+        self.conv = nn.Conv2d(input_dim, output_dim, kernel_size, stride=1, padding=kernel_size // 2, bias=biases)
+        if spec_norm:
+            self.conv = spectral_norm(self.conv)
+
+    def forward(self, inputs):
+        output = inputs
+        output = sum([output[:, :, ::2, ::2], output[:, :, 1::2, ::2],
+                      output[:, :, ::2, 1::2], output[:, :, 1::2, 1::2]]) / 4.
+        return self.conv(output)
+
+
+class UpsampleConv(nn.Module):
+    def __init__(self, input_dim, output_dim, kernel_size=3, biases=True, spec_norm=False):
+        super().__init__()
+        self.conv = nn.Conv2d(input_dim, output_dim, kernel_size, stride=1, padding=kernel_size // 2, bias=biases)
+        if spec_norm:
+            self.conv = spectral_norm(self.conv)
+        self.pixelshuffle = nn.PixelShuffle(upscale_factor=2)
+
+    def forward(self, inputs):
+        output = inputs
+        output = torch.cat([output, output, output, output], dim=1)
+        output = self.pixelshuffle(output)
+        return self.conv(output)
+
+
+class ConditionalResidualBlock(nn.Module):
+    def __init__(self, input_dim, output_dim, num_classes, resample=None, act=nn.ELU(),
+                 normalization=ConditionalBatchNorm2d, adjust_padding=False, dilation=None, spec_norm=False):
+        super().__init__()
+        self.non_linearity = act
+        self.input_dim = input_dim
+        self.output_dim = output_dim
+        self.resample = resample
+        self.normalization = normalization
+        if resample == 'down':
+            if dilation is not None:
+                self.conv1 = dilated_conv3x3(input_dim, input_dim, dilation=dilation, spec_norm=spec_norm)
+                self.normalize2 = normalization(input_dim, num_classes)
+                self.conv2 = dilated_conv3x3(input_dim, output_dim, dilation=dilation, spec_norm=spec_norm)
+                conv_shortcut = partial(dilated_conv3x3, dilation=dilation, spec_norm=spec_norm)
+            else:
+                self.conv1 = conv3x3(input_dim, input_dim, spec_norm=spec_norm)
+                self.normalize2 = normalization(input_dim, num_classes)
+                self.conv2 = ConvMeanPool(input_dim, output_dim, 3, adjust_padding=adjust_padding, spec_norm=spec_norm)
+                conv_shortcut = partial(ConvMeanPool, kernel_size=1, adjust_padding=adjust_padding, spec_norm=spec_norm)
+
+        elif resample is None:
+            if dilation is not None:
+                conv_shortcut = partial(dilated_conv3x3, dilation=dilation, spec_norm=spec_norm)
+                self.conv1 = dilated_conv3x3(input_dim, output_dim, dilation=dilation, spec_norm=spec_norm)
+                self.normalize2 = normalization(output_dim, num_classes)
+                self.conv2 = dilated_conv3x3(output_dim, output_dim, dilation=dilation, spec_norm=spec_norm)
+            else:
+                conv_shortcut = nn.Conv2d
+                self.conv1 = conv3x3(input_dim, output_dim, spec_norm=spec_norm)
+                self.normalize2 = normalization(output_dim, num_classes)
+                self.conv2 = conv3x3(output_dim, output_dim, spec_norm=spec_norm)
+        else:
+            raise Exception('invalid resample value')
+
+        if output_dim != input_dim or resample is not None:
+            self.shortcut = conv_shortcut(input_dim, output_dim)
+
+        self.normalize1 = normalization(input_dim, num_classes)
+
+
+    def forward(self, x, y):
+        output = self.normalize1(x, y)
+        output = self.non_linearity(output)
+        output = self.conv1(output)
+        output = self.normalize2(output, y)
+        output = self.non_linearity(output)
+        output = self.conv2(output)
+
+        if self.output_dim == self.input_dim and self.resample is None:
+            shortcut = x
+        else:
+            shortcut = self.shortcut(x)
+
+        return shortcut + output
+
+
+class ResidualBlock(nn.Module):
+    def __init__(self, input_dim, output_dim, resample=None, act=nn.ELU(),
+                 normalization=nn.BatchNorm2d, adjust_padding=False, dilation=None, spec_norm=False):
+        super().__init__()
+        self.non_linearity = act
+        self.input_dim = input_dim
+        self.output_dim = output_dim
+        self.resample = resample
+        self.normalization = normalization
+        if resample == 'down':
+            if dilation is not None:
+                self.conv1 = dilated_conv3x3(input_dim, input_dim, dilation=dilation, spec_norm=spec_norm)
+                self.normalize2 = normalization(input_dim)
+                self.conv2 = dilated_conv3x3(input_dim, output_dim, dilation=dilation, spec_norm=spec_norm)
+                conv_shortcut = partial(dilated_conv3x3, dilation=dilation, spec_norm=spec_norm)
+            else:
+                self.conv1 = conv3x3(input_dim, input_dim, spec_norm=spec_norm)
+                self.normalize2 = normalization(input_dim)
+                self.conv2 = ConvMeanPool(input_dim, output_dim, 3, adjust_padding=adjust_padding, spec_norm=spec_norm)
+                conv_shortcut = partial(ConvMeanPool, kernel_size=1, adjust_padding=adjust_padding, spec_norm=spec_norm)
+
+        elif resample is None:
+            if dilation is not None:
+                conv_shortcut = partial(dilated_conv3x3, dilation=dilation, spec_norm=spec_norm)
+                self.conv1 = dilated_conv3x3(input_dim, output_dim, dilation=dilation, spec_norm=spec_norm)
+                self.normalize2 = normalization(output_dim)
+                self.conv2 = dilated_conv3x3(output_dim, output_dim, dilation=dilation, spec_norm=spec_norm)
+            else:
+                # conv_shortcut = nn.Conv2d ### Something wierd here.
+                conv_shortcut = partial(conv1x1, spec_norm=spec_norm)
+                self.conv1 = conv3x3(input_dim, output_dim, spec_norm=spec_norm)
+                self.normalize2 = normalization(output_dim)
+                self.conv2 = conv3x3(output_dim, output_dim, spec_norm=spec_norm)
+        else:
+            raise Exception('invalid resample value')
+
+        if output_dim != input_dim or resample is not None:
+            self.shortcut = conv_shortcut(input_dim, output_dim)
+
+        self.normalize1 = normalization(input_dim)
+
+
+    def forward(self, x):
+        output = self.normalize1(x)
+        output = self.non_linearity(output)
+        output = self.conv1(output)
+        output = self.normalize2(output)
+        output = self.non_linearity(output)
+        output = self.conv2(output)
+
+        if self.output_dim == self.input_dim and self.resample is None:
+            shortcut = x
+        else:
+            shortcut = self.shortcut(x)
+
+        return shortcut + output

ncsn/ncsnv2.py

@@ -0,0 +1,314 @@
+import torch.nn as nn
+import numpy as np
+import torch.nn.functional as F
+import torch
+from functools import partial
+from .layers import *
+from .normalization import get_normalization
+
+
+def get_sigmas(config):
+    if config.model.sigma_dist == 'geometric':
+        sigmas = torch.tensor(
+            np.exp(np.linspace(np.log(config.model.sigma_begin), np.log(config.model.sigma_end),
+                               config.model.num_classes))).float().to(config.device)
+    elif config.model.sigma_dist == 'uniform':
+        sigmas = torch.tensor(
+            np.linspace(config.model.sigma_begin, config.model.sigma_end, config.model.num_classes)
+        ).float().to(config.device)
+
+    else:
+        raise NotImplementedError('sigma distribution not supported')
+
+    return sigmas
+
+
+class NCSNv2(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.logit_transform = config.data.logit_transform
+        self.rescaled = config.data.rescaled
+        self.norm = get_normalization(config, conditional=False)
+        self.ngf = ngf = config.model.ngf
+        self.num_classes = num_classes = config.model.num_classes
+
+        self.act = act = get_act(config)
+        self.register_buffer('sigmas', get_sigmas(config))
+        self.config = config
+
+        self.begin_conv = nn.Conv2d(config.data.channels, ngf, 3, stride=1, padding=1)
+
+        self.normalizer = self.norm(ngf, self.num_classes)
+        self.end_conv = nn.Conv2d(ngf, config.data.channels, 3, stride=1, padding=1)
+
+        self.res1 = nn.ModuleList([
+            ResidualBlock(self.ngf, self.ngf, resample=None, act=act,
+                          normalization=self.norm),
+            ResidualBlock(self.ngf, self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res2 = nn.ModuleList([
+            ResidualBlock(self.ngf, 2 * self.ngf, resample='down', act=act,
+                          normalization=self.norm),
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res3 = nn.ModuleList([
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample='down', act=act,
+                          normalization=self.norm, dilation=2),
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                          normalization=self.norm, dilation=2)]
+        )
+
+        if config.data.image_size == 28:
+            self.res4 = nn.ModuleList([
+                ResidualBlock(2 * self.ngf, 2 * self.ngf, resample='down', act=act,
+                              normalization=self.norm, adjust_padding=True, dilation=4),
+                ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                              normalization=self.norm, dilation=4)]
+            )
+        else:
+            self.res4 = nn.ModuleList([
+                ResidualBlock(2 * self.ngf, 2 * self.ngf, resample='down', act=act,
+                              normalization=self.norm, adjust_padding=False, dilation=4),
+                ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                              normalization=self.norm, dilation=4)]
+            )
+
+        self.refine1 = RefineBlock([2 * self.ngf], 2 * self.ngf, act=act, start=True)
+        self.refine2 = RefineBlock([2 * self.ngf, 2 * self.ngf], 2 * self.ngf, act=act)
+        self.refine3 = RefineBlock([2 * self.ngf, 2 * self.ngf], self.ngf, act=act)
+        self.refine4 = RefineBlock([self.ngf, self.ngf], self.ngf, act=act, end=True)
+
+    def _compute_cond_module(self, module, x):
+        for m in module:
+            x = m(x)
+        return x
+
+    def forward(self, x, y):
+        if not self.logit_transform and not self.rescaled:
+            h = 2 * x - 1.
+        else:
+            h = x
+
+        output = self.begin_conv(h)
+
+        layer1 = self._compute_cond_module(self.res1, output)
+        layer2 = self._compute_cond_module(self.res2, layer1)
+        layer3 = self._compute_cond_module(self.res3, layer2)
+        layer4 = self._compute_cond_module(self.res4, layer3)
+
+        ref1 = self.refine1([layer4], layer4.shape[2:])
+        ref2 = self.refine2([layer3, ref1], layer3.shape[2:])
+        ref3 = self.refine3([layer2, ref2], layer2.shape[2:])
+        output = self.refine4([layer1, ref3], layer1.shape[2:])
+
+        output = self.normalizer(output)
+        output = self.act(output)
+        output = self.end_conv(output)
+
+        used_sigmas = self.sigmas[y].view(x.shape[0], *([1] * len(x.shape[1:])))
+
+        output = output / used_sigmas
+
+        return output
+
+
+class NCSNv2Deeper(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.logit_transform = config.data.logit_transform
+        self.rescaled = config.data.rescaled
+        self.norm = get_normalization(config, conditional=False)
+        self.ngf = ngf = config.model.ngf
+        self.num_classes = config.model.num_classes
+        self.act = act = get_act(config)
+        self.register_buffer('sigmas', get_sigmas(config))
+        self.config = config
+
+        self.begin_conv = nn.Conv2d(config.data.channels, ngf, 3, stride=1, padding=1)
+        self.normalizer = self.norm(ngf, self.num_classes)
+
+        self.end_conv = nn.Conv2d(ngf, config.data.channels, 3, stride=1, padding=1)
+
+        self.res1 = nn.ModuleList([
+            ResidualBlock(self.ngf, self.ngf, resample=None, act=act,
+                          normalization=self.norm),
+            ResidualBlock(self.ngf, self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res2 = nn.ModuleList([
+            ResidualBlock(self.ngf, 2 * self.ngf, resample='down', act=act,
+                          normalization=self.norm),
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res3 = nn.ModuleList([
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample='down', act=act,
+                          normalization=self.norm),
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res4 = nn.ModuleList([
+            ResidualBlock(2 * self.ngf, 4 * self.ngf, resample='down', act=act,
+                          normalization=self.norm, dilation=2),
+            ResidualBlock(4 * self.ngf, 4 * self.ngf, resample=None, act=act,
+                          normalization=self.norm, dilation=2)]
+        )
+
+        self.res5 = nn.ModuleList([
+            ResidualBlock(4 * self.ngf, 4 * self.ngf, resample='down', act=act,
+                          normalization=self.norm, dilation=4),
+            ResidualBlock(4 * self.ngf, 4 * self.ngf, resample=None, act=act,
+                          normalization=self.norm, dilation=4)]
+        )
+
+        self.refine1 = RefineBlock([4 * self.ngf], 4 * self.ngf, act=act, start=True)
+        self.refine2 = RefineBlock([4 * self.ngf, 4 * self.ngf], 2 * self.ngf, act=act)
+        self.refine3 = RefineBlock([2 * self.ngf, 2 * self.ngf], 2 * self.ngf, act=act)
+        self.refine4 = RefineBlock([2 * self.ngf, 2 * self.ngf], self.ngf, act=act)
+        self.refine5 = RefineBlock([self.ngf, self.ngf], self.ngf, act=act, end=True)
+
+    def _compute_cond_module(self, module, x):
+        for m in module:
+            x = m(x)
+        return x
+
+    def forward(self, x, y):
+        if not self.logit_transform and not self.rescaled:
+            h = 2 * x - 1.
+        else:
+            h = x
+
+        output = self.begin_conv(h)
+
+        layer1 = self._compute_cond_module(self.res1, output)
+        layer2 = self._compute_cond_module(self.res2, layer1)
+        layer3 = self._compute_cond_module(self.res3, layer2)
+        layer4 = self._compute_cond_module(self.res4, layer3)
+        layer5 = self._compute_cond_module(self.res5, layer4)
+
+        ref1 = self.refine1([layer5], layer5.shape[2:])
+        ref2 = self.refine2([layer4, ref1], layer4.shape[2:])
+        ref3 = self.refine3([layer3, ref2], layer3.shape[2:])
+        ref4 = self.refine4([layer2, ref3], layer2.shape[2:])
+        output = self.refine5([layer1, ref4], layer1.shape[2:])
+
+        output = self.normalizer(output)
+        output = self.act(output)
+        output = self.end_conv(output)
+
+        used_sigmas = self.sigmas[y].view(x.shape[0], *([1] * len(x.shape[1:])))
+
+        output = output / used_sigmas
+
+        return output
+
+
+class NCSNv2Deepest(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.logit_transform = config.data.logit_transform
+        self.rescaled = config.data.rescaled
+        self.norm = get_normalization(config, conditional=False)
+        self.ngf = ngf = config.model.ngf
+        self.num_classes = config.model.num_classes
+        self.act = act = get_act(config)
+        self.register_buffer('sigmas', get_sigmas(config))
+        self.config = config
+
+        self.begin_conv = nn.Conv2d(config.data.channels, ngf, 3, stride=1, padding=1)
+        self.normalizer = self.norm(ngf, self.num_classes)
+
+        self.end_conv = nn.Conv2d(ngf, config.data.channels, 3, stride=1, padding=1)
+
+        self.res1 = nn.ModuleList([
+            ResidualBlock(self.ngf, self.ngf, resample=None, act=act,
+                          normalization=self.norm),
+            ResidualBlock(self.ngf, self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res2 = nn.ModuleList([
+            ResidualBlock(self.ngf, 2 * self.ngf, resample='down', act=act,
+                          normalization=self.norm),
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res3 = nn.ModuleList([
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample='down', act=act,
+                          normalization=self.norm),
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res31 = nn.ModuleList([
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample='down', act=act,
+                          normalization=self.norm),
+            ResidualBlock(2 * self.ngf, 2 * self.ngf, resample=None, act=act,
+                          normalization=self.norm)]
+        )
+
+        self.res4 = nn.ModuleList([
+            ResidualBlock(2 * self.ngf, 4 * self.ngf, resample='down', act=act,
+                          normalization=self.norm, dilation=2),
+            ResidualBlock(4 * self.ngf, 4 * self.ngf, resample=None, act=act,
+                          normalization=self.norm, dilation=2)]
+        )
+
+        self.res5 = nn.ModuleList([
+            ResidualBlock(4 * self.ngf, 4 * self.ngf, resample='down', act=act,
+                          normalization=self.norm, dilation=4),
+            ResidualBlock(4 * self.ngf, 4 * self.ngf, resample=None, act=act,
+                          normalization=self.norm, dilation=4)]
+        )
+
+        self.refine1 = RefineBlock([4 * self.ngf], 4 * self.ngf, act=act, start=True)
+        self.refine2 = RefineBlock([4 * self.ngf, 4 * self.ngf], 2 * self.ngf, act=act)
+        self.refine3 = RefineBlock([2 * self.ngf, 2 * self.ngf], 2 * self.ngf, act=act)
+        self.refine31 = RefineBlock([2 * self.ngf, 2 * self.ngf], 2 * self.ngf, act=act)
+        self.refine4 = RefineBlock([2 * self.ngf, 2 * self.ngf], self.ngf, act=act)
+        self.refine5 = RefineBlock([self.ngf, self.ngf], self.ngf, act=act, end=True)
+
+    def _compute_cond_module(self, module, x):
+        for m in module:
+            x = m(x)
+        return x
+
+    def forward(self, x, y):
+        if not self.logit_transform and not self.rescaled:
+            h = 2 * x - 1.
+        else:
+            h = x
+
+        output = self.begin_conv(h)
+
+        layer1 = self._compute_cond_module(self.res1, output)
+        layer2 = self._compute_cond_module(self.res2, layer1)
+        layer3 = self._compute_cond_module(self.res3, layer2)
+        layer31 = self._compute_cond_module(self.res31, layer3)
+        layer4 = self._compute_cond_module(self.res4, layer31)
+        layer5 = self._compute_cond_module(self.res5, layer4)
+
+        ref1 = self.refine1([layer5], layer5.shape[2:])
+        ref2 = self.refine2([layer4, ref1], layer4.shape[2:])
+        ref31 = self.refine31([layer31, ref2], layer31.shape[2:])
+        ref3 = self.refine3([layer3, ref31], layer3.shape[2:])
+        ref4 = self.refine4([layer2, ref3], layer2.shape[2:])
+        output = self.refine5([layer1, ref4], layer1.shape[2:])
+
+        output = self.normalizer(output)
+        output = self.act(output)
+        output = self.end_conv(output)
+
+        used_sigmas = self.sigmas[y].view(x.shape[0], *([1] * len(x.shape[1:])))
+
+        output = output / used_sigmas
+
+        return output

ncsn/normalization.py

@@ -0,0 +1,208 @@
+import torch
+import torch.nn as nn
+
+
+def get_normalization(config, conditional=True):
+    norm = config.model.normalization
+    if conditional:
+        if norm == 'NoneNorm':
+            return ConditionalNoneNorm2d
+        elif norm == 'InstanceNorm++':
+            return ConditionalInstanceNorm2dPlus
+        elif norm == 'InstanceNorm':
+            return ConditionalInstanceNorm2d
+        elif norm == 'BatchNorm':
+            return ConditionalBatchNorm2d
+        elif norm == 'VarianceNorm':
+            return ConditionalVarianceNorm2d
+        else:
+            raise NotImplementedError("{} does not exist!".format(norm))
+    else:
+        if norm == 'BatchNorm':
+            return nn.BatchNorm2d
+        elif norm == 'InstanceNorm':
+            return nn.InstanceNorm2d
+        elif norm == 'InstanceNorm++':
+            return InstanceNorm2dPlus
+        elif norm == 'VarianceNorm':
+            return VarianceNorm2d
+        elif norm == 'NoneNorm':
+            return NoneNorm2d
+        elif norm is None:
+            return None
+        else:
+            raise NotImplementedError("{} does not exist!".format(norm))
+
+class ConditionalBatchNorm2d(nn.Module):
+    def __init__(self, num_features, num_classes, bias=True):
+        super().__init__()
+        self.num_features = num_features
+        self.bias = bias
+        self.bn = nn.BatchNorm2d(num_features, affine=False)
+        if self.bias:
+            self.embed = nn.Embedding(num_classes, num_features * 2)
+            self.embed.weight.data[:, :num_features].uniform_()  # Initialise scale at N(1, 0.02)
+            self.embed.weight.data[:, num_features:].zero_()  # Initialise bias at 0
+        else:
+            self.embed = nn.Embedding(num_classes, num_features)
+            self.embed.weight.data.uniform_()
+
+    def forward(self, x, y):
+        out = self.bn(x)
+        if self.bias:
+            gamma, beta = self.embed(y).chunk(2, dim=1)
+            out = gamma.view(-1, self.num_features, 1, 1) * out + beta.view(-1, self.num_features, 1, 1)
+        else:
+            gamma = self.embed(y)
+            out = gamma.view(-1, self.num_features, 1, 1) * out
+        return out
+
+
+class ConditionalInstanceNorm2d(nn.Module):
+    def __init__(self, num_features, num_classes, bias=True):
+        super().__init__()
+        self.num_features = num_features
+        self.bias = bias
+        self.instance_norm = nn.InstanceNorm2d(num_features, affine=False, track_running_stats=False)
+        if bias:
+            self.embed = nn.Embedding(num_classes, num_features * 2)
+            self.embed.weight.data[:, :num_features].uniform_()  # Initialise scale at N(1, 0.02)
+            self.embed.weight.data[:, num_features:].zero_()  # Initialise bias at 0
+        else:
+            self.embed = nn.Embedding(num_classes, num_features)
+            self.embed.weight.data.uniform_()
+
+    def forward(self, x, y):
+        h = self.instance_norm(x)
+        if self.bias:
+            gamma, beta = self.embed(y).chunk(2, dim=-1)
+            out = gamma.view(-1, self.num_features, 1, 1) * h + beta.view(-1, self.num_features, 1, 1)
+        else:
+            gamma = self.embed(y)
+            out = gamma.view(-1, self.num_features, 1, 1) * h
+        return out
+
+
+class ConditionalVarianceNorm2d(nn.Module):
+    def __init__(self, num_features, num_classes, bias=False):
+        super().__init__()
+        self.num_features = num_features
+        self.bias = bias
+        self.embed = nn.Embedding(num_classes, num_features)
+        self.embed.weight.data.normal_(1, 0.02)
+
+    def forward(self, x, y):
+        vars = torch.var(x, dim=(2, 3), keepdim=True)
+        h = x / torch.sqrt(vars + 1e-5)
+
+        gamma = self.embed(y)
+        out = gamma.view(-1, self.num_features, 1, 1) * h
+        return out
+
+
+class VarianceNorm2d(nn.Module):
+    def __init__(self, num_features, bias=False):
+        super().__init__()
+        self.num_features = num_features
+        self.bias = bias
+        self.alpha = nn.Parameter(torch.zeros(num_features))
+        self.alpha.data.normal_(1, 0.02)
+
+    def forward(self, x):
+        vars = torch.var(x, dim=(2, 3), keepdim=True)
+        h = x / torch.sqrt(vars + 1e-5)
+
+        out = self.alpha.view(-1, self.num_features, 1, 1) * h
+        return out
+
+
+class ConditionalNoneNorm2d(nn.Module):
+    def __init__(self, num_features, num_classes, bias=True):
+        super().__init__()
+        self.num_features = num_features
+        self.bias = bias
+        if bias:
+            self.embed = nn.Embedding(num_classes, num_features * 2)
+            self.embed.weight.data[:, :num_features].uniform_()  # Initialise scale at N(1, 0.02)
+            self.embed.weight.data[:, num_features:].zero_()  # Initialise bias at 0
+        else:
+            self.embed = nn.Embedding(num_classes, num_features)
+            self.embed.weight.data.uniform_()
+
+    def forward(self, x, y):
+        if self.bias:
+            gamma, beta = self.embed(y).chunk(2, dim=-1)
+            out = gamma.view(-1, self.num_features, 1, 1) * x + beta.view(-1, self.num_features, 1, 1)
+        else:
+            gamma = self.embed(y)
+            out = gamma.view(-1, self.num_features, 1, 1) * x
+        return out
+
+
+class NoneNorm2d(nn.Module):
+    def __init__(self, num_features, bias=True):
+        super().__init__()
+
+    def forward(self, x):
+        return x
+
+
+class InstanceNorm2dPlus(nn.Module):
+    def __init__(self, num_features, bias=True):
+        super().__init__()
+        self.num_features = num_features
+        self.bias = bias
+        self.instance_norm = nn.InstanceNorm2d(num_features, affine=False, track_running_stats=False)
+        self.alpha = nn.Parameter(torch.zeros(num_features))
+        self.gamma = nn.Parameter(torch.zeros(num_features))
+        self.alpha.data.normal_(1, 0.02)
+        self.gamma.data.normal_(1, 0.02)
+        if bias:
+            self.beta = nn.Parameter(torch.zeros(num_features))
+
+    def forward(self, x):
+        means = torch.mean(x, dim=(2, 3))
+        m = torch.mean(means, dim=-1, keepdim=True)
+        v = torch.var(means, dim=-1, keepdim=True)
+        means = (means - m) / (torch.sqrt(v + 1e-5))
+        h = self.instance_norm(x)
+
+        if self.bias:
+            h = h + means[..., None, None] * self.alpha[..., None, None]
+            out = self.gamma.view(-1, self.num_features, 1, 1) * h + self.beta.view(-1, self.num_features, 1, 1)
+        else:
+            h = h + means[..., None, None] * self.alpha[..., None, None]
+            out = self.gamma.view(-1, self.num_features, 1, 1) * h
+        return out
+
+
+class ConditionalInstanceNorm2dPlus(nn.Module):
+    def __init__(self, num_features, num_classes, bias=True):
+        super().__init__()
+        self.num_features = num_features
+        self.bias = bias
+        self.instance_norm = nn.InstanceNorm2d(num_features, affine=False, track_running_stats=False)
+        if bias:
+            self.embed = nn.Embedding(num_classes, num_features * 3)
+            self.embed.weight.data[:, :2 * num_features].normal_(1, 0.02)  # Initialise scale at N(1, 0.02)
+            self.embed.weight.data[:, 2 * num_features:].zero_()  # Initialise bias at 0
+        else:
+            self.embed = nn.Embedding(num_classes, 2 * num_features)
+            self.embed.weight.data.normal_(1, 0.02)
+
+    def forward(self, x, y):
+        means = torch.mean(x, dim=(2, 3))
+        m = torch.mean(means, dim=-1, keepdim=True)
+        v = torch.var(means, dim=-1, keepdim=True)
+        means = (means - m) / (torch.sqrt(v + 1e-5))
+        h = self.instance_norm(x)
+
+        if self.bias:
+            gamma, alpha, beta = self.embed(y).chunk(3, dim=-1)
+            h = h + means[..., None, None] * alpha[..., None, None]
+            out = gamma.view(-1, self.num_features, 1, 1) * h + beta.view(-1, self.num_features, 1, 1)
+        else:
+            gamma, alpha = self.embed(y).chunk(2, dim=-1)
+            h = h + means[..., None, None] * alpha[..., None, None]
+            out = gamma.view(-1, self.num_features, 1, 1) * h
+        return out

pose.py

@@ -0,0 +1,120 @@
+import numpy as np
+from numpy import sin, cos
+from math import pi as π
+from my3d import camera_pose
+from my.config import BaseConf
+import random
+
+
+def get_K(H, W, FoV_x):
+    FoV_x = FoV_x / 180 * π  # to rad
+    f = 1 / np.tan(FoV_x / 2) * (W / 2)
+
+    K = np.array([
+        [f, 0, -(W/2 - 0.5)],
+        [0, -f, -(H/2 - 0.5)],
+        [0, 0, -1]
+    ])
+    return K
+
+
+SIDEVIEW_PROMPTS = [
+    "front view of", "side view of", "backside view of", "side view of"
+]
+
+TOPVIEW_PROMPT = "overhead view of"
+
+
+def train_eye_with_prompts(r, n):
+    hs = np.random.rand(n) * 360
+    vs = np.random.rand(n) * np.deg2rad(100)
+    vs = np.clip(vs, 1e-2, π-1e-2)
+
+    prompts = []
+    v_thresh = np.deg2rad(30)
+    for i in range(n):
+        _p = ""
+        if vs[i] < v_thresh:
+            _p = TOPVIEW_PROMPT
+        else:
+            _a = hs[i]
+            _a = (_a + 45) % 360
+            _quad = int(_a // 90)
+            _p = SIDEVIEW_PROMPTS[_quad]
+        prompts.append(_p)
+
+    θ = np.deg2rad(hs)
+    # φ = v
+    φ = np.arccos(1 - 2 * (vs / π))
+
+    eyes = np.zeros((n, 3))
+
+    eyes[:, 0] = r * sin(φ) * cos(π-θ)  # x
+    eyes[:, 2] = r * sin(φ) * sin(π-θ)  # z
+    eyes[:, 1] = r * cos(φ)  # y
+
+    return eyes, prompts
+
+
+def spiral_poses(
+    radius, height,
+    num_steps=20, num_rounds=1,
+    center=np.array([0, 0, 0]), up=np.array([0, 1, 0]),
+):
+    eyes = []
+    for i in range(num_steps):
+        ratio = (i + 1) / num_steps
+        Δy = height * (1 - ratio)
+
+        θ = ratio * (360 * num_rounds)
+        θ = θ / 180 * π
+        # _r = max(radius * ratio, 0.5)
+        _r = max(radius * sin(ratio * π / 2), 0.5)
+        Δx, Δz = _r * np.array([np.cos(θ), np.sin(θ)])
+        eyes.append(center + [Δx, Δy, Δz])
+
+    poses = [
+        camera_pose(e, center - e, up) for e in eyes
+    ]
+    return poses
+
+
+class PoseConfig(BaseConf):
+    rend_hw: int = 64
+    FoV: float = 60.0
+    R: float = 1.5
+
+    def make(self):
+        cfgs = self.dict()
+        hw = cfgs.pop("rend_hw")
+        cfgs["H"] = hw
+        cfgs["W"] = hw
+        return Poser(**cfgs)
+
+
+class Poser():
+    def __init__(self, H, W, FoV, R):
+        self.H, self.W = H, W
+        self.R = R
+        self.K = get_K(H, W, FoV)
+
+    def sample_train(self, n):
+        eyes, prompts = train_eye_with_prompts(r=self.R, n=n)
+        up = np.array([0, 1, 0])
+        poses = [
+            camera_pose(e, -e, up) for e in eyes
+        ]
+        poses = np.stack(poses, 0)
+        # FoV during training: [40,70]
+        random_Ks = [
+            get_K(self.H, self.W, random.random() * 30 + 40)
+            for i in range(len(poses))
+            # self.K for i in range(len(poses))
+        ]
+        # return self.K, poses, prompts
+        return random_Ks, poses, prompts
+
+    def sample_test(self, n):
+        poses = spiral_poses(self.R, self.R, n, num_rounds=3)
+        poses = np.stack(poses, axis=0)
+        return self.K, poses

release/diffusion_ckpts/guided_ddpm/models/lsun_bedroom.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9faf136dc2375dcdb392b35cee9ca9dca1fd5257b2f3358613136395ec39231
+size 2211383297

release/diffusion_ckpts/guided_ddpm/models/lsun_ffhq.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e409993ae12fc4cb8cd61aba7352c1bc0af0735e2debdd4b3c609280c8dc448b
+size 2211370791

release/diffusion_ckpts/stable_diffusion/sd-v1-5.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1441589a6f3c5a53f5f54d0975a18a7feb7cdf0b0dee276dfc3331ae376a053
+size 7703807346

requirements.txt

@@ -0,0 +1,21 @@
+pydantic
+tqdm
+click
+easydict
+tabulate
+imageio
+einops
+matplotlib
+omegaconf==2.1.1
+torchmetrics==0.6.0
+pytorch-lightning==1.4.2
+transformers
+kornia==0.6.0
+git+https:///github.com/openai/CLIP.git#egg=clip
+imageio[ffmpeg]
+imageio[pyav]
+--extra-index-url https://download.pytorch.org/whl/cu116
+torch
+torchvision
+torchaudio
+-e git+https://github.com/CompVis/taming-transformers.git#egg=taming-transformers

run_img_sampling.py

@@ -0,0 +1,235 @@
+from pathlib import Path
+import numpy as np
+import torch
+
+from misc import torch_samps_to_imgs
+from adapt import Karras, ScoreAdapter, power_schedule
+from adapt_gddpm import GuidedDDPM
+from adapt_ncsn import NCSN as _NCSN
+# from adapt_vesde import VESDE  # not included to prevent import conflicts
+from adapt_sd import StableDiffusion
+
+from my.utils import tqdm, EventStorage, HeartBeat, EarlyLoopBreak
+from my.config import BaseConf, dispatch
+from my.utils.seed import seed_everything
+
+
+class GDDPM(BaseConf):
+    """Guided DDPM from OpenAI"""
+    model:          str = "m_lsun_256"
+    lsun_cat:       str = "bedroom"
+    imgnet_cat:     int = -1
+
+    def make(self):
+        args = self.dict()
+        model = GuidedDDPM(**args)
+        return model
+
+
+class SD(BaseConf):
+    """Stable Diffusion"""
+    variant:        str = "v1"
+    v2_highres:     bool = False
+    prompt:         str = "a photograph of an astronaut riding a horse"
+    scale:          float = 3.0  # classifier free guidance scale
+    precision:      str = 'autocast'
+
+    def make(self):
+        args = self.dict()
+        model = StableDiffusion(**args)
+        return model
+
+
+class SDE(BaseConf):
+    def make(self):
+        args = self.dict()
+        model = VESDE(**args)
+        return model
+
+
+class NCSN(BaseConf):
+    def make(self):
+        args = self.dict()
+        model = _NCSN(**args)
+        return model
+
+
+class KarrasGen(BaseConf):
+    family:         str = "gddpm"
+    gddpm:          GDDPM = GDDPM()
+    sd:             SD = SD()
+    # sde:            SDE = SDE()
+    ncsn:           NCSN = NCSN()
+
+    batch_size:     int = 10
+    num_images:     int = 1250
+    num_t:          int = 40
+    σ_max:          float = 80.0
+    heun:           bool = True
+    langevin:       bool = False
+    cls_scaling:    float = 1.0  # classifier guidance scaling
+
+    def run(self):
+        args = self.dict()
+        family = args.pop("family")
+        model = getattr(self, family).make()
+        self.karras_generate(model, **args)
+
+    @staticmethod
+    def karras_generate(
+        model: ScoreAdapter,
+        batch_size, num_images, σ_max, num_t, langevin, heun, cls_scaling,
+        **kwargs
+    ):
+        del kwargs  # removed extra args
+        num_batches = num_images // batch_size
+
+        fuse = EarlyLoopBreak(5)
+        with tqdm(total=num_batches) as pbar, \
+            HeartBeat(pbar) as hbeat, \
+                EventStorage() as metric:
+
+            all_imgs = []
+
+            for _ in range(num_batches):
+                if fuse.on_break():
+                    break
+
+                pipeline = Karras.inference(
+                    model, batch_size, num_t,
+                    init_xs=None, heun=heun, σ_max=σ_max,
+                    langevin=langevin, cls_scaling=cls_scaling
+                )
+
+                for imgs in tqdm(pipeline, total=num_t+1, disable=False):
+                    # _std = imgs.std().item()
+                    # print(_std)
+                    hbeat.beat()
+                    pass
+
+                if isinstance(model, StableDiffusion):
+                    imgs = model.decode(imgs)
+
+                imgs = torch_samps_to_imgs(imgs, uncenter=model.samps_centered())
+                all_imgs.append(imgs)
+
+                pbar.update()
+
+            all_imgs = np.concatenate(all_imgs, axis=0)
+            metric.put_artifact("imgs", ".npy", lambda fn: np.save(fn, all_imgs))
+            metric.step()
+            hbeat.done()
+
+
+class SMLDGen(BaseConf):
+    family:         str = "ncsn"
+    gddpm:          GDDPM = GDDPM()
+    # sde:            SDE = SDE()
+    ncsn:           NCSN = NCSN()
+
+    batch_size:     int = 16
+    num_images:     int = 16
+    num_stages:     int = 80
+    num_steps:      int = 15
+    σ_max:          float = 80.0
+    ε:              float = 1e-5
+
+    def run(self):
+        args = self.dict()
+        family = args.pop("family")
+        model = getattr(self, family).make()
+        self.smld_generate(model, **args)
+
+    @staticmethod
+    def smld_generate(
+        model: ScoreAdapter,
+        batch_size, num_images, num_stages, num_steps, σ_max, ε,
+        **kwargs
+    ):
+        num_batches = num_images // batch_size
+        σs = power_schedule(σ_max, model.σ_min, num_stages)
+        σs = [model.snap_t_to_nearest_tick(σ)[0] for σ in σs]
+
+        fuse = EarlyLoopBreak(5)
+        with tqdm(total=num_batches) as pbar, \
+            HeartBeat(pbar) as hbeat, \
+                EventStorage() as metric:
+
+            all_imgs = []
+
+            for _ in range(num_batches):
+                if fuse.on_break():
+                    break
+
+                init_xs = torch.rand(batch_size, *model.data_shape(), device=model.device)
+                if model.samps_centered():
+                    init_xs = init_xs * 2 - 1  # [0, 1] -> [-1, 1]
+
+                pipeline = smld_inference(
+                    model, σs, num_steps, ε, init_xs
+                )
+
+                for imgs in tqdm(pipeline, total=(num_stages * num_steps)+1, disable=False):
+                    pbar.set_description(f"{imgs.max().item():.3f}")
+                    metric.put_scalars(
+                        max=imgs.max().item(), min=imgs.min().item(), std=imgs.std().item()
+                    )
+                    metric.step()
+                    hbeat.beat()
+
+                pbar.update()
+                imgs = torch_samps_to_imgs(imgs, uncenter=model.samps_centered())
+                all_imgs.append(imgs)
+
+            all_imgs = np.concatenate(all_imgs, axis=0)
+            metric.put_artifact("imgs", ".npy", lambda fn: np.save(fn, all_imgs))
+            metric.step()
+            hbeat.done()
+
+
+def smld_inference(model, σs, num_steps, ε, init_xs):
+    from math import sqrt
+    # not doing conditioning or cls guidance; for gddpm only lsun works; fine.
+
+    xs = init_xs
+    yield xs
+
+    for i in range(len(σs)):
+        α_i = ε * ((σs[i] / σs[-1]) ** 2)
+        for _ in range(num_steps):
+            grad = model.score(xs, σs[i])
+            z = torch.randn_like(xs)
+            xs = xs + α_i * grad + sqrt(2 * α_i) * z
+            yield xs
+
+
+def load_np_imgs(fname):
+    fname = Path(fname)
+    data = np.load(fname)
+    if fname.suffix == ".npz":
+        imgs = data['arr_0']
+    else:
+        imgs = data
+    return imgs
+
+
+def visualize(max_n_imgs=16):
+    import torchvision.utils as vutils
+    from imageio import imwrite
+    from einops import rearrange
+
+    all_imgs = load_np_imgs("imgs/step_0.npy")
+
+    imgs = all_imgs[:max_n_imgs]
+    imgs = rearrange(imgs, "N H W C -> N C H W", C=3)
+    imgs = torch.from_numpy(imgs)
+    pane = vutils.make_grid(imgs, padding=2, nrow=4)
+    pane = rearrange(pane, "C H W -> H W C", C=3)
+    pane = pane.numpy()
+    imwrite("preview.jpg", pane)
+
+
+if __name__ == "__main__":
+    seed_everything(0)
+    dispatch(KarrasGen)
+    visualize(16)

run_nerf.py

@@ -0,0 +1,62 @@
+from typing import List
+from pydantic import validator
+
+from my.config import BaseConf, SingleOrList, dispatch
+from my.utils.seed import seed_everything
+
+import numpy as np
+from voxnerf.vox import VOXRF_REGISTRY
+from voxnerf.pipelines import train
+
+
+class VoxConfig(BaseConf):
+    model_type:                 str = "VoxRF"
+    bbox_len:                   float = 1.5
+    grid_size:                  SingleOrList(int) = [128, 128, 128]
+    step_ratio:                 float = 0.5
+    density_shift:              float = -10.
+    ray_march_weight_thres:     float = 0.0001
+    c:                          int = 3
+    blend_bg_texture:           bool = False
+    bg_texture_hw:              int = 64
+
+    @validator("grid_size")
+    def check_gsize(cls, grid_size):
+        if isinstance(grid_size, int):
+            return [grid_size, ] * 3
+        else:
+            assert len(grid_size) == 3
+            return grid_size
+
+    def make(self):
+        params = self.dict()
+        m_type = params.pop("model_type")
+        model_fn = VOXRF_REGISTRY.get(m_type)
+
+        radius = params.pop('bbox_len')
+        aabb = radius * np.array([
+            [-1, -1, -1],
+            [1, 1, 1]
+        ])
+        model = model_fn(aabb=aabb, **params)
+        return model
+
+
+class TrainerConfig(BaseConf):
+    model:      VoxConfig = VoxConfig()
+    scene:      str = "lego"
+    n_epoch:    int = 2
+    bs:         int = 4096
+    lr:         float = 0.02
+
+    def run(self):
+        args = self.dict()
+        args.pop("model")
+
+        model = self.model.make()
+        train(model, **args)
+
+
+if __name__ == "__main__":
+    seed_everything(0)
+    dispatch(TrainerConfig)

run_sjc.py

@@ -0,0 +1,298 @@
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+from einops import rearrange
+from imageio import imwrite
+from pydantic import validator
+
+from my.utils import (
+    tqdm, EventStorage, HeartBeat, EarlyLoopBreak,
+    get_event_storage, get_heartbeat, read_stats
+)
+from my.config import BaseConf, dispatch, optional_load_config
+from my.utils.seed import seed_everything
+
+from adapt import ScoreAdapter, karras_t_schedule
+from run_img_sampling import GDDPM, SD, StableDiffusion
+from misc import torch_samps_to_imgs
+from pose import PoseConfig
+
+from run_nerf import VoxConfig
+from voxnerf.utils import every
+from voxnerf.render import (
+    as_torch_tsrs, rays_from_img, ray_box_intersect, render_ray_bundle
+)
+from voxnerf.vis import stitch_vis, bad_vis as nerf_vis
+
+
+device_glb = torch.device("cuda")
+
+
+def tsr_stats(tsr):
+    return {
+        "mean": tsr.mean().item(),
+        "std": tsr.std().item(),
+        "max": tsr.max().item(),
+    }
+
+
+class SJC(BaseConf):
+    family:     str = "sd"
+    gddpm:      GDDPM = GDDPM()
+    sd:         SD = SD(
+        variant="v1",
+        prompt="A high quality photo of a delicious burger",
+        scale=100.0
+    )
+    lr:         float = 0.05
+    n_steps:    int = 10000
+    vox:        VoxConfig = VoxConfig(
+        model_type="V_SD", grid_size=100, density_shift=-1.0, c=3,
+        blend_bg_texture=True, bg_texture_hw=4,
+        bbox_len=1.0
+    )
+    pose:       PoseConfig = PoseConfig(rend_hw=64, FoV=60.0, R=1.5)
+
+    emptiness_scale:    int = 10
+    emptiness_weight:   int = 1e4
+    emptiness_step:     float = 0.5
+    emptiness_multiplier: float = 20.0
+
+    depth_weight:       int = 0
+
+    var_red:     bool = True
+
+    @validator("vox")
+    def check_vox(cls, vox_cfg, values):
+        family = values['family']
+        if family == "sd":
+            vox_cfg.c = 4
+        return vox_cfg
+
+    def run(self):
+        cfgs = self.dict()
+
+        family = cfgs.pop("family")
+        model = getattr(self, family).make()
+
+        cfgs.pop("vox")
+        vox = self.vox.make()
+
+        cfgs.pop("pose")
+        poser = self.pose.make()
+
+        sjc_3d(**cfgs, poser=poser, model=model, vox=vox)
+
+
+def sjc_3d(
+    poser, vox, model: ScoreAdapter,
+    lr, n_steps, emptiness_scale, emptiness_weight, emptiness_step, emptiness_multiplier,
+    depth_weight, var_red, **kwargs
+):
+    del kwargs
+
+    assert model.samps_centered()
+    _, target_H, target_W = model.data_shape()
+    bs = 1
+    aabb = vox.aabb.T.cpu().numpy()
+    vox = vox.to(device_glb)
+    opt = torch.optim.Adamax(vox.opt_params(), lr=lr)
+
+    H, W = poser.H, poser.W
+    Ks, poses, prompt_prefixes = poser.sample_train(n_steps)
+
+    ts = model.us[30:-10]
+    fuse = EarlyLoopBreak(5)
+
+    same_noise = torch.randn(1, 4, H, W, device=model.device).repeat(bs, 1, 1, 1)
+
+    with tqdm(total=n_steps) as pbar, \
+        HeartBeat(pbar) as hbeat, \
+            EventStorage() as metric:
+        for i in range(n_steps):
+            if fuse.on_break():
+                break
+
+            p = f"{prompt_prefixes[i]} {model.prompt}"
+            score_conds = model.prompts_emb([p])
+
+            y, depth, ws = render_one_view(vox, aabb, H, W, Ks[i], poses[i], return_w=True)
+
+            if isinstance(model, StableDiffusion):
+                pass
+            else:
+                y = torch.nn.functional.interpolate(y, (target_H, target_W), mode='bilinear')
+
+            opt.zero_grad()
+
+            with torch.no_grad():
+                chosen_σs = np.random.choice(ts, bs, replace=False)
+                chosen_σs = chosen_σs.reshape(-1, 1, 1, 1)
+                chosen_σs = torch.as_tensor(chosen_σs, device=model.device, dtype=torch.float32)
+                # chosen_σs = us[i]
+
+                noise = torch.randn(bs, *y.shape[1:], device=model.device)
+
+                zs = y + chosen_σs * noise
+                Ds = model.denoise(zs, chosen_σs, **score_conds)
+
+                if var_red:
+                    grad = (Ds - y) / chosen_σs
+                else:
+                    grad = (Ds - zs) / chosen_σs
+
+                grad = grad.mean(0, keepdim=True)
+
+            y.backward(-grad, retain_graph=True)
+
+            if depth_weight > 0:
+                center_depth = depth[7:-7, 7:-7]
+                border_depth_mean = (depth.sum() - center_depth.sum()) / (64*64-50*50)
+                center_depth_mean = center_depth.mean()
+                depth_diff = center_depth_mean - border_depth_mean
+                depth_loss = - torch.log(depth_diff + 1e-12)
+                depth_loss = depth_weight * depth_loss
+                depth_loss.backward(retain_graph=True)
+
+            emptiness_loss = torch.log(1 + emptiness_scale * ws).mean()
+            emptiness_loss = emptiness_weight * emptiness_loss
+            if emptiness_step * n_steps <= i:
+                emptiness_loss *= emptiness_multiplier
+            emptiness_loss.backward()
+
+            opt.step()
+
+            metric.put_scalars(**tsr_stats(y))
+
+            if every(pbar, percent=1):
+                with torch.no_grad():
+                    if isinstance(model, StableDiffusion):
+                        y = model.decode(y)
+                    vis_routine(metric, y, depth)
+
+            # if every(pbar, step=2500):
+            #     metric.put_artifact(
+            #         "ckpt", ".pt", lambda fn: torch.save(vox.state_dict(), fn)
+            #     )
+            #     with EventStorage("test"):
+            #         evaluate(model, vox, poser)
+
+            metric.step()
+            pbar.update()
+            pbar.set_description(p)
+            hbeat.beat()
+
+        metric.put_artifact(
+            "ckpt", ".pt", lambda fn: torch.save(vox.state_dict(), fn)
+        )
+        with EventStorage("test"):
+            evaluate(model, vox, poser)
+
+        metric.step()
+
+        hbeat.done()
+
+
+@torch.no_grad()
+def evaluate(score_model, vox, poser):
+    H, W = poser.H, poser.W
+    vox.eval()
+    K, poses = poser.sample_test(100)
+
+    fuse = EarlyLoopBreak(5)
+    metric = get_event_storage()
+    hbeat = get_heartbeat()
+
+    aabb = vox.aabb.T.cpu().numpy()
+    vox = vox.to(device_glb)
+
+    num_imgs = len(poses)
+
+    for i in (pbar := tqdm(range(num_imgs))):
+        if fuse.on_break():
+            break
+
+        pose = poses[i]
+        y, depth = render_one_view(vox, aabb, H, W, K, pose)
+        if isinstance(score_model, StableDiffusion):
+            y = score_model.decode(y)
+        vis_routine(metric, y, depth)
+
+        metric.step()
+        hbeat.beat()
+
+    metric.flush_history()
+
+    metric.put_artifact(
+        "view_seq", ".mp4",
+        lambda fn: stitch_vis(fn, read_stats(metric.output_dir, "view")[1])
+    )
+
+    metric.step()
+
+
+def render_one_view(vox, aabb, H, W, K, pose, return_w=False):
+    N = H * W
+    ro, rd = rays_from_img(H, W, K, pose)
+    ro, rd, t_min, t_max = scene_box_filter(ro, rd, aabb)
+    assert len(ro) == N, "for now all pixels must be in"
+    ro, rd, t_min, t_max = as_torch_tsrs(vox.device, ro, rd, t_min, t_max)
+    rgbs, depth, weights = render_ray_bundle(vox, ro, rd, t_min, t_max)
+
+    rgbs = rearrange(rgbs, "(h w) c -> 1 c h w", h=H, w=W)
+    depth = rearrange(depth, "(h w) 1 -> h w", h=H, w=W)
+    if return_w:
+        return rgbs, depth, weights
+    else:
+        return rgbs, depth
+
+
+def scene_box_filter(ro, rd, aabb):
+    _, t_min, t_max = ray_box_intersect(ro, rd, aabb)
+    # do not render what's behind the ray origin
+    t_min, t_max = np.maximum(t_min, 0), np.maximum(t_max, 0)
+    return ro, rd, t_min, t_max
+
+
+def vis_routine(metric, y, depth):
+    pane = nerf_vis(y, depth, final_H=256)
+    im = torch_samps_to_imgs(y)[0]
+    depth = depth.cpu().numpy()
+    metric.put_artifact("view", ".png", lambda fn: imwrite(fn, pane))
+    metric.put_artifact("img", ".png", lambda fn: imwrite(fn, im))
+    metric.put_artifact("depth", ".npy", lambda fn: np.save(fn, depth))
+
+
+def evaluate_ckpt():
+    cfg = optional_load_config(fname="full_config.yml")
+    assert len(cfg) > 0, "can't find cfg file"
+    mod = SJC(**cfg)
+
+    family = cfg.pop("family")
+    model: ScoreAdapter = getattr(mod, family).make()
+    vox = mod.vox.make()
+    poser = mod.pose.make()
+
+    pbar = tqdm(range(1))
+
+    with EventStorage(), HeartBeat(pbar):
+        ckpt_fname = latest_ckpt()
+        state = torch.load(ckpt_fname, map_location="cpu")
+        vox.load_state_dict(state)
+        vox.to(device_glb)
+
+        with EventStorage("test"):
+            evaluate(model, vox, poser)
+
+
+def latest_ckpt():
+    ts, ys = read_stats("./", "ckpt")
+    assert len(ys) > 0
+    return ys[-1]
+
+
+if __name__ == "__main__":
+    seed_everything(0)
+    dispatch(SJC)
+    # evaluate_ckpt()