update changes

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,167 @@
+gradio_cached_examples/
+
+# https://github.com/github/gitignore/blob/main/Python.gitignore
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+requirements-dev.lock

.python-version

@@ -0,0 +1 @@
+3.12.2

LICENSE

@@ -0,0 +1,235 @@
+MIT License
+
+Copyright (c) 2024 Lagon Technologies
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This software contains portions of third party software provided under other licenses:
+
+src/esrgan_model.py (x)
+===================
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+(x) Modified from https://github.com/philz1337x/clarity-upscaler
+    which is a copy of https://github.com/AUTOMATIC1111/stable-diffusion-webui
+    which is a copy of https://github.com/victorca25/iNNfer
+    which is a copy of https://github.com/xinntao/ESRGAN

pyproject.toml

@@ -0,0 +1,54 @@
+[project]
+name = "enhancer"
+version = "0.1.0"
+description = "Finegrain Image Enhancer"
+authors = [
+    { name = "Laurent Fainsin", email = "laurent@lagon.tech" }
+]
+dependencies = [
+    "gradio>=4.42.0",
+    "pillow>=10.4.0",
+    "gradio-imageslider>=0.0.20",
+    "pillow-heif>=0.18.0",
+    "refiners @ git+https://github.com/finegrain-ai/refiners",
+    "spaces>=0.29.3",
+    "numpy<2.0.0",
+]
+readme = "README.md"
+requires-python = ">= 3.12, <3.13"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.rye]
+managed = true
+dev-dependencies = []
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/enhancer"]
+
+[tool.ruff]
+src = ["src"]  # https://docs.astral.sh/ruff/settings/#src
+exclude = ["esrgan_model.py"]
+line-length = 120
+target-version = "py312"
+
+[tool.ruff.lint]
+select = [
+  "E",   # pycodestyle errors
+  "W",   # pycodestyle warnings
+  "F",   # pyflakes
+  "UP",  # pyupgrade
+  "A",   # flake8-builtins
+  "B",   # flake8-bugbear
+  "Q",   # flake8-quotes
+  "I",   # isort
+]
+
+[tool.pyright]
+include = ["src"]
+exclude = ["**/__pycache__"]

requirements.lock

@@ -0,0 +1,231 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+#   universal: false
+
+-e file:.
+aiofiles==23.2.1
+    # via gradio
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.4.0
+    # via gradio
+    # via httpx
+    # via starlette
+certifi==2024.8.30
+    # via httpcore
+    # via httpx
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via typer
+    # via uvicorn
+contourpy==1.3.0
+    # via matplotlib
+cycler==0.12.1
+    # via matplotlib
+fastapi==0.113.0
+    # via gradio
+ffmpy==0.4.0
+    # via gradio
+filelock==3.15.4
+    # via huggingface-hub
+    # via torch
+    # via triton
+fonttools==4.53.1
+    # via matplotlib
+fsspec==2024.9.0
+    # via gradio-client
+    # via huggingface-hub
+    # via torch
+gradio==4.42.0
+    # via enhancer
+    # via gradio-imageslider
+    # via spaces
+gradio-client==1.3.0
+    # via gradio
+gradio-imageslider==0.0.20
+    # via enhancer
+h11==0.14.0
+    # via httpcore
+    # via uvicorn
+httpcore==1.0.5
+    # via httpx
+httpx==0.27.2
+    # via gradio
+    # via gradio-client
+    # via spaces
+huggingface-hub==0.24.6
+    # via gradio
+    # via gradio-client
+idna==3.8
+    # via anyio
+    # via httpx
+    # via requests
+importlib-resources==6.4.4
+    # via gradio
+jaxtyping==0.2.34
+    # via refiners
+jinja2==3.1.4
+    # via gradio
+    # via torch
+kiwisolver==1.4.7
+    # via matplotlib
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via gradio
+    # via jinja2
+matplotlib==3.9.2
+    # via gradio
+mdurl==0.1.2
+    # via markdown-it-py
+mpmath==1.3.0
+    # via sympy
+networkx==3.3
+    # via torch
+numpy==1.26.4
+    # via contourpy
+    # via enhancer
+    # via gradio
+    # via matplotlib
+    # via pandas
+    # via refiners
+nvidia-cublas-cu12==12.1.3.1
+    # via nvidia-cudnn-cu12
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-cuda-cupti-cu12==12.1.105
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.1.105
+    # via torch
+nvidia-cuda-runtime-cu12==12.1.105
+    # via torch
+nvidia-cudnn-cu12==9.1.0.70
+    # via torch
+nvidia-cufft-cu12==11.0.2.54
+    # via torch
+nvidia-curand-cu12==10.3.2.106
+    # via torch
+nvidia-cusolver-cu12==11.4.5.107
+    # via torch
+nvidia-cusparse-cu12==12.1.0.106
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-nccl-cu12==2.20.5
+    # via torch
+nvidia-nvjitlink-cu12==12.6.68
+    # via nvidia-cusolver-cu12
+    # via nvidia-cusparse-cu12
+nvidia-nvtx-cu12==12.1.105
+    # via torch
+orjson==3.10.7
+    # via gradio
+packaging==24.1
+    # via gradio
+    # via gradio-client
+    # via huggingface-hub
+    # via matplotlib
+    # via refiners
+    # via spaces
+pandas==2.2.2
+    # via gradio
+pillow==10.4.0
+    # via enhancer
+    # via gradio
+    # via gradio-imageslider
+    # via matplotlib
+    # via pillow-heif
+    # via refiners
+pillow-heif==0.18.0
+    # via enhancer
+psutil==5.9.8
+    # via spaces
+pydantic==2.8.2
+    # via fastapi
+    # via gradio
+    # via spaces
+pydantic-core==2.20.1
+    # via pydantic
+pydub==0.25.1
+    # via gradio
+pygments==2.18.0
+    # via rich
+pyparsing==3.1.4
+    # via matplotlib
+python-dateutil==2.9.0.post0
+    # via matplotlib
+    # via pandas
+python-multipart==0.0.9
+    # via gradio
+pytz==2024.1
+    # via pandas
+pyyaml==6.0.2
+    # via gradio
+    # via huggingface-hub
+refiners @ git+https://github.com/finegrain-ai/refiners@cf247a1b20609479565618f49bf70c8aa65a7cfd
+    # via enhancer
+requests==2.32.3
+    # via huggingface-hub
+    # via spaces
+rich==13.8.0
+    # via typer
+ruff==0.6.4
+    # via gradio
+safetensors==0.4.5
+    # via refiners
+semantic-version==2.10.0
+    # via gradio
+setuptools==74.1.2
+    # via torch
+shellingham==1.5.4
+    # via typer
+six==1.16.0
+    # via python-dateutil
+sniffio==1.3.1
+    # via anyio
+    # via httpx
+spaces==0.30.1
+    # via enhancer
+starlette==0.38.4
+    # via fastapi
+sympy==1.13.2
+    # via torch
+tomlkit==0.12.0
+    # via gradio
+torch==2.4.1
+    # via refiners
+tqdm==4.66.5
+    # via huggingface-hub
+triton==3.0.0
+    # via torch
+typeguard==2.13.3
+    # via jaxtyping
+typer==0.12.5
+    # via gradio
+typing-extensions==4.12.2
+    # via fastapi
+    # via gradio
+    # via gradio-client
+    # via huggingface-hub
+    # via pydantic
+    # via pydantic-core
+    # via spaces
+    # via torch
+    # via typer
+tzdata==2024.1
+    # via pandas
+urllib3==2.2.2
+    # via gradio
+    # via requests
+uvicorn==0.30.6
+    # via gradio
+websockets==12.0
+    # via gradio-client

requirements.txt

@@ -0,0 +1,6 @@
+git+https://github.com/finegrain-ai/refiners@a5d3c2971b84f6faa4762b1cf5a07f4f812bb1f5
+gradio_imageslider==0.0.20
+spaces==0.28.3
+numpy<2.0.0
+pillow>=10.4.0
+pillow-heif>=0.18.0

src/app.py

@@ -0,0 +1,299 @@
+from pathlib import Path
+
+import gradio as gr
+import pillow_heif
+import spaces
+import torch
+from gradio_imageslider import ImageSlider
+from huggingface_hub import hf_hub_download
+from PIL import Image
+from refiners.fluxion.utils import manual_seed
+from refiners.foundationals.latent_diffusion import Solver, solvers
+
+from enhancer import ESRGANUpscaler, ESRGANUpscalerCheckpoints
+
+pillow_heif.register_heif_opener()
+pillow_heif.register_avif_opener()
+
+TITLE = """
+<center>
+
+  <h1 style="font-size: 1.5rem; margin-bottom: 0.5rem;">
+    Image Enhancer Powered By Refiners
+  </h1>
+
+  <div style="
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    gap: 0.5rem;
+    margin-bottom: 0.5rem;
+    font-size: 1.25rem;
+    flex-wrap: wrap;
+  ">
+    <a href="https://blog.finegrain.ai/posts/reproducing-clarity-upscaler/" target="_blank">[Blog Post]</a>
+    <a href="https://github.com/finegrain-ai/refiners" target="_blank">[Refiners]</a>
+    <a href="https://finegrain.ai/" target="_blank">[Finegrain]</a>
+    <a href="https://huggingface.co/spaces/finegrain/finegrain-object-eraser" target="_blank">
+        [Finegrain Object Eraser]
+    </a>
+    <a href="https://huggingface.co/spaces/finegrain/finegrain-object-cutter" target="_blank">
+        [Finegrain Object Cutter]
+    </a>
+  </div>
+
+  <p>
+    Turn low resolution images into high resolution versions with added generated details (your image will be modified).
+  </p>
+
+  <p>
+    This space is powered by Refiners, our open source micro-framework for simple foundation model adaptation.
+    If you enjoyed it, please consider starring Refiners on GitHub!
+  </p>
+
+  <a href="https://github.com/finegrain-ai/refiners" target="_blank">
+    <img src="https://img.shields.io/github/stars/finegrain-ai/refiners?style=social" />
+  </a>
+
+</center>
+"""
+
+CHECKPOINTS = ESRGANUpscalerCheckpoints(
+    unet=Path(
+        hf_hub_download(
+            repo_id="refiners/juggernaut.reborn.sd1_5.unet",
+            filename="model.safetensors",
+            revision="347d14c3c782c4959cc4d1bb1e336d19f7dda4d2",
+        )
+    ),
+    clip_text_encoder=Path(
+        hf_hub_download(
+            repo_id="refiners/juggernaut.reborn.sd1_5.text_encoder",
+            filename="model.safetensors",
+            revision="744ad6a5c0437ec02ad826df9f6ede102bb27481",
+        )
+    ),
+    lda=Path(
+        hf_hub_download(
+            repo_id="refiners/juggernaut.reborn.sd1_5.autoencoder",
+            filename="model.safetensors",
+            revision="3c1aae3fc3e03e4a2b7e0fa42b62ebb64f1a4c19",
+        )
+    ),
+    controlnet_tile=Path(
+        hf_hub_download(
+            repo_id="refiners/controlnet.sd1_5.tile",
+            filename="model.safetensors",
+            revision="48ced6ff8bfa873a8976fa467c3629a240643387",
+        )
+    ),
+    esrgan=Path(
+        hf_hub_download(
+            repo_id="philz1337x/upscaler",
+            filename="4x-UltraSharp.pth",
+            revision="011deacac8270114eb7d2eeff4fe6fa9a837be70",
+        )
+    ),
+    negative_embedding=Path(
+        hf_hub_download(
+            repo_id="philz1337x/embeddings",
+            filename="JuggernautNegative-neg.pt",
+            revision="203caa7e9cc2bc225031a4021f6ab1ded283454a",
+        )
+    ),
+    negative_embedding_key="string_to_param.*",
+    loras={
+        "more_details": Path(
+            hf_hub_download(
+                repo_id="philz1337x/loras",
+                filename="more_details.safetensors",
+                revision="a3802c0280c0d00c2ab18d37454a8744c44e474e",
+            )
+        ),
+        "sdxl_render": Path(
+            hf_hub_download(
+                repo_id="philz1337x/loras",
+                filename="SDXLrender_v2.0.safetensors",
+                revision="a3802c0280c0d00c2ab18d37454a8744c44e474e",
+            )
+        ),
+    },
+)
+
+# initialize the enhancer, on the cpu
+DEVICE_CPU = torch.device("cpu")
+DTYPE = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32
+enhancer = ESRGANUpscaler(checkpoints=CHECKPOINTS, device=DEVICE_CPU, dtype=DTYPE)
+
+# "move" the enhancer to the gpu, this is handled by Zero GPU
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+enhancer.to(device=DEVICE, dtype=DTYPE)
+
+
+@spaces.GPU
+def process(
+    input_image: Image.Image,
+    prompt: str = "masterpiece, best quality, highres",
+    negative_prompt: str = "worst quality, low quality, normal quality",
+    seed: int = 42,
+    upscale_factor: int = 2,
+    controlnet_scale: float = 0.6,
+    controlnet_decay: float = 1.0,
+    condition_scale: int = 6,
+    tile_width: int = 112,
+    tile_height: int = 144,
+    denoise_strength: float = 0.35,
+    num_inference_steps: int = 18,
+    solver: str = "DDIM",
+) -> tuple[Image.Image, Image.Image]:
+    manual_seed(seed)
+
+    solver_type: type[Solver] = getattr(solvers, solver)
+
+    enhanced_image = enhancer.upscale(
+        image=input_image,
+        prompt=prompt,
+        negative_prompt=negative_prompt,
+        upscale_factor=upscale_factor,
+        controlnet_scale=controlnet_scale,
+        controlnet_scale_decay=controlnet_decay,
+        condition_scale=condition_scale,
+        tile_size=(tile_height, tile_width),
+        denoise_strength=denoise_strength,
+        num_inference_steps=num_inference_steps,
+        loras_scale={"more_details": 0.5, "sdxl_render": 1.0},
+        solver_type=solver_type,
+    )
+
+    return (input_image, enhanced_image)
+
+
+with gr.Blocks() as demo:
+    gr.HTML(TITLE)
+
+    with gr.Row():
+        with gr.Column():
+            input_image = gr.Image(type="pil", label="Input Image")
+            run_button = gr.ClearButton(components=None, value="Enhance Image")
+        with gr.Column():
+            output_slider = ImageSlider(label="Before / After")
+            run_button.add(output_slider)
+
+    with gr.Accordion("Advanced Options", open=False):
+        prompt = gr.Textbox(
+            label="Prompt",
+            placeholder="masterpiece, best quality, highres",
+        )
+        negative_prompt = gr.Textbox(
+            label="Negative Prompt",
+            placeholder="worst quality, low quality, normal quality",
+        )
+        seed = gr.Slider(
+            minimum=0,
+            maximum=10_000,
+            value=42,
+            step=1,
+            label="Seed",
+        )
+        upscale_factor = gr.Slider(
+            minimum=1,
+            maximum=4,
+            value=2,
+            step=0.2,
+            label="Upscale Factor",
+        )
+        controlnet_scale = gr.Slider(
+            minimum=0,
+            maximum=1.5,
+            value=0.6,
+            step=0.1,
+            label="ControlNet Scale",
+        )
+        controlnet_decay = gr.Slider(
+            minimum=0.5,
+            maximum=1,
+            value=1.0,
+            step=0.025,
+            label="ControlNet Scale Decay",
+        )
+        condition_scale = gr.Slider(
+            minimum=2,
+            maximum=20,
+            value=6,
+            step=1,
+            label="Condition Scale",
+        )
+        tile_width = gr.Slider(
+            minimum=64,
+            maximum=200,
+            value=112,
+            step=1,
+            label="Latent Tile Width",
+        )
+        tile_height = gr.Slider(
+            minimum=64,
+            maximum=200,
+            value=144,
+            step=1,
+            label="Latent Tile Height",
+        )
+        denoise_strength = gr.Slider(
+            minimum=0,
+            maximum=1,
+            value=0.35,
+            step=0.1,
+            label="Denoise Strength",
+        )
+        num_inference_steps = gr.Slider(
+            minimum=1,
+            maximum=30,
+            value=18,
+            step=1,
+            label="Number of Inference Steps",
+        )
+        solver = gr.Radio(
+            choices=["DDIM", "DPMSolver"],
+            value="DDIM",
+            label="Solver",
+        )
+
+    run_button.click(
+        fn=process,
+        inputs=[
+            input_image,
+            prompt,
+            negative_prompt,
+            seed,
+            upscale_factor,
+            controlnet_scale,
+            controlnet_decay,
+            condition_scale,
+            tile_width,
+            tile_height,
+            denoise_strength,
+            num_inference_steps,
+            solver,
+        ],
+        outputs=output_slider,
+    )
+
+    gr.Examples(
+        examples=[
+            "examples/kara-eads-L7EwHkq1B2s-unsplash.jpg",
+            "examples/clarity_bird.webp",
+            "examples/edgar-infocus-gJH8AqpiSEU-unsplash.jpg",
+            "examples/jeremy-wallace-_XjW3oN8UOE-unsplash.jpg",
+            "examples/karina-vorozheeva-rW-I87aPY5Y-unsplash.jpg",
+            "examples/karographix-photography-hIaOPjYCEj4-unsplash.jpg",
+            "examples/melissa-walker-horn-gtDYwUIr9Vg-unsplash.jpg",
+            "examples/ryoji-iwata-X53e51WfjlE-unsplash.jpg",
+            "examples/tadeusz-lakota-jggQZkITXng-unsplash.jpg",
+        ],
+        inputs=[input_image],
+        outputs=output_slider,
+        fn=process,
+        cache_examples="lazy",
+        run_on_click=False,
+    )
+
+demo.launch(share=False)

src/enhancer.py

@@ -0,0 +1,39 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import torch
+from PIL import Image
+from refiners.foundationals.latent_diffusion.stable_diffusion_1.multi_upscaler import (
+    MultiUpscaler,
+    UpscalerCheckpoints,
+)
+
+from esrgan_model import UpscalerESRGAN
+
+
+@dataclass(kw_only=True)
+class ESRGANUpscalerCheckpoints(UpscalerCheckpoints):
+    esrgan: Path
+
+
+class ESRGANUpscaler(MultiUpscaler):
+    def __init__(
+        self,
+        checkpoints: ESRGANUpscalerCheckpoints,
+        device: torch.device,
+        dtype: torch.dtype,
+    ) -> None:
+        super().__init__(checkpoints=checkpoints, device=device, dtype=dtype)
+        self.esrgan = UpscalerESRGAN(checkpoints.esrgan, device=self.device, dtype=self.dtype)
+        self.esrgan.to(device=device, dtype=dtype)
+
+    def to(self, device: torch.device, dtype: torch.dtype):
+        self.esrgan.to(device=device, dtype=dtype)
+        self.sd = self.sd.to(device=device, dtype=dtype)
+        self.device = device
+        self.dtype = dtype
+
+    def pre_upscale(self, image: Image.Image, upscale_factor: float, **_: Any) -> Image.Image:
+        image = self.esrgan.upscale_with_tiling(image)
+        return super().pre_upscale(image=image, upscale_factor=upscale_factor / 4)

src/esrgan_model.py

@@ -0,0 +1,1068 @@
+# type: ignore
+"""
+Modified from https://github.com/philz1337x/clarity-upscaler
+which is a copy of https://github.com/AUTOMATIC1111/stable-diffusion-webui
+which is a copy of https://github.com/victorca25/iNNfer
+which is a copy of https://github.com/xinntao/ESRGAN
+"""
+
+import math
+import os
+from collections import OrderedDict, namedtuple
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from PIL import Image
+
+####################
+# RRDBNet Generator
+####################
+
+
+class RRDBNet(nn.Module):
+    def __init__(
+        self,
+        in_nc,
+        out_nc,
+        nf,
+        nb,
+        nr=3,
+        gc=32,
+        upscale=4,
+        norm_type=None,
+        act_type="leakyrelu",
+        mode="CNA",
+        upsample_mode="upconv",
+        convtype="Conv2D",
+        finalact=None,
+        gaussian_noise=False,
+        plus=False,
+    ):
+        super(RRDBNet, self).__init__()
+        n_upscale = int(math.log(upscale, 2))
+        if upscale == 3:
+            n_upscale = 1
+
+        self.resrgan_scale = 0
+        if in_nc % 16 == 0:
+            self.resrgan_scale = 1
+        elif in_nc != 4 and in_nc % 4 == 0:
+            self.resrgan_scale = 2
+
+        fea_conv = conv_block(in_nc, nf, kernel_size=3, norm_type=None, act_type=None, convtype=convtype)
+        rb_blocks = [
+            RRDB(
+                nf,
+                nr,
+                kernel_size=3,
+                gc=32,
+                stride=1,
+                bias=1,
+                pad_type="zero",
+                norm_type=norm_type,
+                act_type=act_type,
+                mode="CNA",
+                convtype=convtype,
+                gaussian_noise=gaussian_noise,
+                plus=plus,
+            )
+            for _ in range(nb)
+        ]
+        LR_conv = conv_block(
+            nf,
+            nf,
+            kernel_size=3,
+            norm_type=norm_type,
+            act_type=None,
+            mode=mode,
+            convtype=convtype,
+        )
+
+        if upsample_mode == "upconv":
+            upsample_block = upconv_block
+        elif upsample_mode == "pixelshuffle":
+            upsample_block = pixelshuffle_block
+        else:
+            raise NotImplementedError(f"upsample mode [{upsample_mode}] is not found")
+        if upscale == 3:
+            upsampler = upsample_block(nf, nf, 3, act_type=act_type, convtype=convtype)
+        else:
+            upsampler = [upsample_block(nf, nf, act_type=act_type, convtype=convtype) for _ in range(n_upscale)]
+        HR_conv0 = conv_block(nf, nf, kernel_size=3, norm_type=None, act_type=act_type, convtype=convtype)
+        HR_conv1 = conv_block(nf, out_nc, kernel_size=3, norm_type=None, act_type=None, convtype=convtype)
+
+        outact = act(finalact) if finalact else None
+
+        self.model = sequential(
+            fea_conv,
+            ShortcutBlock(sequential(*rb_blocks, LR_conv)),
+            *upsampler,
+            HR_conv0,
+            HR_conv1,
+            outact,
+        )
+
+    def forward(self, x, outm=None):
+        if self.resrgan_scale == 1:
+            feat = pixel_unshuffle(x, scale=4)
+        elif self.resrgan_scale == 2:
+            feat = pixel_unshuffle(x, scale=2)
+        else:
+            feat = x
+
+        return self.model(feat)
+
+
+class RRDB(nn.Module):
+    """
+    Residual in Residual Dense Block
+    (ESRGAN: Enhanced Super-Resolution Generative Adversarial Networks)
+    """
+
+    def __init__(
+        self,
+        nf,
+        nr=3,
+        kernel_size=3,
+        gc=32,
+        stride=1,
+        bias=1,
+        pad_type="zero",
+        norm_type=None,
+        act_type="leakyrelu",
+        mode="CNA",
+        convtype="Conv2D",
+        spectral_norm=False,
+        gaussian_noise=False,
+        plus=False,
+    ):
+        super(RRDB, self).__init__()
+        # This is for backwards compatibility with existing models
+        if nr == 3:
+            self.RDB1 = ResidualDenseBlock_5C(
+                nf,
+                kernel_size,
+                gc,
+                stride,
+                bias,
+                pad_type,
+                norm_type,
+                act_type,
+                mode,
+                convtype,
+                spectral_norm=spectral_norm,
+                gaussian_noise=gaussian_noise,
+                plus=plus,
+            )
+            self.RDB2 = ResidualDenseBlock_5C(
+                nf,
+                kernel_size,
+                gc,
+                stride,
+                bias,
+                pad_type,
+                norm_type,
+                act_type,
+                mode,
+                convtype,
+                spectral_norm=spectral_norm,
+                gaussian_noise=gaussian_noise,
+                plus=plus,
+            )
+            self.RDB3 = ResidualDenseBlock_5C(
+                nf,
+                kernel_size,
+                gc,
+                stride,
+                bias,
+                pad_type,
+                norm_type,
+                act_type,
+                mode,
+                convtype,
+                spectral_norm=spectral_norm,
+                gaussian_noise=gaussian_noise,
+                plus=plus,
+            )
+        else:
+            RDB_list = [
+                ResidualDenseBlock_5C(
+                    nf,
+                    kernel_size,
+                    gc,
+                    stride,
+                    bias,
+                    pad_type,
+                    norm_type,
+                    act_type,
+                    mode,
+                    convtype,
+                    spectral_norm=spectral_norm,
+                    gaussian_noise=gaussian_noise,
+                    plus=plus,
+                )
+                for _ in range(nr)
+            ]
+            self.RDBs = nn.Sequential(*RDB_list)
+
+    def forward(self, x):
+        if hasattr(self, "RDB1"):
+            out = self.RDB1(x)
+            out = self.RDB2(out)
+            out = self.RDB3(out)
+        else:
+            out = self.RDBs(x)
+        return out * 0.2 + x
+
+
+class ResidualDenseBlock_5C(nn.Module):
+    """
+    Residual Dense Block
+    The core module of paper: (Residual Dense Network for Image Super-Resolution, CVPR 18)
+    Modified options that can be used:
+        - "Partial Convolution based Padding" arXiv:1811.11718
+        - "Spectral normalization" arXiv:1802.05957
+        - "ICASSP 2020 - ESRGAN+ : Further Improving ESRGAN" N. C.
+            {Rakotonirina} and A. {Rasoanaivo}
+    """
+
+    def __init__(
+        self,
+        nf=64,
+        kernel_size=3,
+        gc=32,
+        stride=1,
+        bias=1,
+        pad_type="zero",
+        norm_type=None,
+        act_type="leakyrelu",
+        mode="CNA",
+        convtype="Conv2D",
+        spectral_norm=False,
+        gaussian_noise=False,
+        plus=False,
+    ):
+        super(ResidualDenseBlock_5C, self).__init__()
+
+        self.noise = GaussianNoise() if gaussian_noise else None
+        self.conv1x1 = conv1x1(nf, gc) if plus else None
+
+        self.conv1 = conv_block(
+            nf,
+            gc,
+            kernel_size,
+            stride,
+            bias=bias,
+            pad_type=pad_type,
+            norm_type=norm_type,
+            act_type=act_type,
+            mode=mode,
+            convtype=convtype,
+            spectral_norm=spectral_norm,
+        )
+        self.conv2 = conv_block(
+            nf + gc,
+            gc,
+            kernel_size,
+            stride,
+            bias=bias,
+            pad_type=pad_type,
+            norm_type=norm_type,
+            act_type=act_type,
+            mode=mode,
+            convtype=convtype,
+            spectral_norm=spectral_norm,
+        )
+        self.conv3 = conv_block(
+            nf + 2 * gc,
+            gc,
+            kernel_size,
+            stride,
+            bias=bias,
+            pad_type=pad_type,
+            norm_type=norm_type,
+            act_type=act_type,
+            mode=mode,
+            convtype=convtype,
+            spectral_norm=spectral_norm,
+        )
+        self.conv4 = conv_block(
+            nf + 3 * gc,
+            gc,
+            kernel_size,
+            stride,
+            bias=bias,
+            pad_type=pad_type,
+            norm_type=norm_type,
+            act_type=act_type,
+            mode=mode,
+            convtype=convtype,
+            spectral_norm=spectral_norm,
+        )
+        if mode == "CNA":
+            last_act = None
+        else:
+            last_act = act_type
+        self.conv5 = conv_block(
+            nf + 4 * gc,
+            nf,
+            3,
+            stride,
+            bias=bias,
+            pad_type=pad_type,
+            norm_type=norm_type,
+            act_type=last_act,
+            mode=mode,
+            convtype=convtype,
+            spectral_norm=spectral_norm,
+        )
+
+    def forward(self, x):
+        x1 = self.conv1(x)
+        x2 = self.conv2(torch.cat((x, x1), 1))
+        if self.conv1x1:
+            x2 = x2 + self.conv1x1(x)
+        x3 = self.conv3(torch.cat((x, x1, x2), 1))
+        x4 = self.conv4(torch.cat((x, x1, x2, x3), 1))
+        if self.conv1x1:
+            x4 = x4 + x2
+        x5 = self.conv5(torch.cat((x, x1, x2, x3, x4), 1))
+        if self.noise:
+            return self.noise(x5.mul(0.2) + x)
+        else:
+            return x5 * 0.2 + x
+
+
+####################
+# ESRGANplus
+####################
+
+
+class GaussianNoise(nn.Module):
+    def __init__(self, sigma=0.1, is_relative_detach=False):
+        super().__init__()
+        self.sigma = sigma
+        self.is_relative_detach = is_relative_detach
+        self.noise = torch.tensor(0, dtype=torch.float)
+
+    def forward(self, x):
+        if self.training and self.sigma != 0:
+            self.noise = self.noise.to(device=x.device, dtype=x.device)
+            scale = self.sigma * x.detach() if self.is_relative_detach else self.sigma * x
+            sampled_noise = self.noise.repeat(*x.size()).normal_() * scale
+            x = x + sampled_noise
+        return x
+
+
+def conv1x1(in_planes, out_planes, stride=1):
+    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
+
+
+####################
+# SRVGGNetCompact
+####################
+
+
+class SRVGGNetCompact(nn.Module):
+    """A compact VGG-style network structure for super-resolution.
+    This class is copied from https://github.com/xinntao/Real-ESRGAN
+    """
+
+    def __init__(
+        self,
+        num_in_ch=3,
+        num_out_ch=3,
+        num_feat=64,
+        num_conv=16,
+        upscale=4,
+        act_type="prelu",
+    ):
+        super(SRVGGNetCompact, self).__init__()
+        self.num_in_ch = num_in_ch
+        self.num_out_ch = num_out_ch
+        self.num_feat = num_feat
+        self.num_conv = num_conv
+        self.upscale = upscale
+        self.act_type = act_type
+
+        self.body = nn.ModuleList()
+        # the first conv
+        self.body.append(nn.Conv2d(num_in_ch, num_feat, 3, 1, 1))
+        # the first activation
+        if act_type == "relu":
+            activation = nn.ReLU(inplace=True)
+        elif act_type == "prelu":
+            activation = nn.PReLU(num_parameters=num_feat)
+        elif act_type == "leakyrelu":
+            activation = nn.LeakyReLU(negative_slope=0.1, inplace=True)
+        self.body.append(activation)
+
+        # the body structure
+        for _ in range(num_conv):
+            self.body.append(nn.Conv2d(num_feat, num_feat, 3, 1, 1))
+            # activation
+            if act_type == "relu":
+                activation = nn.ReLU(inplace=True)
+            elif act_type == "prelu":
+                activation = nn.PReLU(num_parameters=num_feat)
+            elif act_type == "leakyrelu":
+                activation = nn.LeakyReLU(negative_slope=0.1, inplace=True)
+            self.body.append(activation)
+
+        # the last conv
+        self.body.append(nn.Conv2d(num_feat, num_out_ch * upscale * upscale, 3, 1, 1))
+        # upsample
+        self.upsampler = nn.PixelShuffle(upscale)
+
+    def forward(self, x):
+        out = x
+        for i in range(0, len(self.body)):
+            out = self.body[i](out)
+
+        out = self.upsampler(out)
+        # add the nearest upsampled image, so that the network learns the residual
+        base = F.interpolate(x, scale_factor=self.upscale, mode="nearest")
+        out += base
+        return out
+
+
+####################
+# Upsampler
+####################
+
+
+class Upsample(nn.Module):
+    r"""Upsamples a given multi-channel 1D (temporal), 2D (spatial) or 3D (volumetric) data.
+    The input data is assumed to be of the form
+    `minibatch x channels x [optional depth] x [optional height] x width`.
+    """
+
+    def __init__(self, size=None, scale_factor=None, mode="nearest", align_corners=None):
+        super(Upsample, self).__init__()
+        if isinstance(scale_factor, tuple):
+            self.scale_factor = tuple(float(factor) for factor in scale_factor)
+        else:
+            self.scale_factor = float(scale_factor) if scale_factor else None
+        self.mode = mode
+        self.size = size
+        self.align_corners = align_corners
+
+    def forward(self, x):
+        return nn.functional.interpolate(
+            x,
+            size=self.size,
+            scale_factor=self.scale_factor,
+            mode=self.mode,
+            align_corners=self.align_corners,
+        )
+
+    def extra_repr(self):
+        if self.scale_factor is not None:
+            info = f"scale_factor={self.scale_factor}"
+        else:
+            info = f"size={self.size}"
+        info += f", mode={self.mode}"
+        return info
+
+
+def pixel_unshuffle(x, scale):
+    """Pixel unshuffle.
+    Args:
+        x (Tensor): Input feature with shape (b, c, hh, hw).
+        scale (int): Downsample ratio.
+    Returns:
+        Tensor: the pixel unshuffled feature.
+    """
+    b, c, hh, hw = x.size()
+    out_channel = c * (scale**2)
+    assert hh % scale == 0 and hw % scale == 0
+    h = hh // scale
+    w = hw // scale
+    x_view = x.view(b, c, h, scale, w, scale)
+    return x_view.permute(0, 1, 3, 5, 2, 4).reshape(b, out_channel, h, w)
+
+
+def pixelshuffle_block(
+    in_nc,
+    out_nc,
+    upscale_factor=2,
+    kernel_size=3,
+    stride=1,
+    bias=True,
+    pad_type="zero",
+    norm_type=None,
+    act_type="relu",
+    convtype="Conv2D",
+):
+    """
+    Pixel shuffle layer
+    (Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional
+    Neural Network, CVPR17)
+    """
+    conv = conv_block(
+        in_nc,
+        out_nc * (upscale_factor**2),
+        kernel_size,
+        stride,
+        bias=bias,
+        pad_type=pad_type,
+        norm_type=None,
+        act_type=None,
+        convtype=convtype,
+    )
+    pixel_shuffle = nn.PixelShuffle(upscale_factor)
+
+    n = norm(norm_type, out_nc) if norm_type else None
+    a = act(act_type) if act_type else None
+    return sequential(conv, pixel_shuffle, n, a)
+
+
+def upconv_block(
+    in_nc,
+    out_nc,
+    upscale_factor=2,
+    kernel_size=3,
+    stride=1,
+    bias=True,
+    pad_type="zero",
+    norm_type=None,
+    act_type="relu",
+    mode="nearest",
+    convtype="Conv2D",
+):
+    """Upconv layer"""
+    upscale_factor = (1, upscale_factor, upscale_factor) if convtype == "Conv3D" else upscale_factor
+    upsample = Upsample(scale_factor=upscale_factor, mode=mode)
+    conv = conv_block(
+        in_nc,
+        out_nc,
+        kernel_size,
+        stride,
+        bias=bias,
+        pad_type=pad_type,
+        norm_type=norm_type,
+        act_type=act_type,
+        convtype=convtype,
+    )
+    return sequential(upsample, conv)
+
+
+####################
+# Basic blocks
+####################
+
+
+def make_layer(basic_block, num_basic_block, **kwarg):
+    """Make layers by stacking the same blocks.
+    Args:
+        basic_block (nn.module): nn.module class for basic block. (block)
+        num_basic_block (int): number of blocks. (n_layers)
+    Returns:
+        nn.Sequential: Stacked blocks in nn.Sequential.
+    """
+    layers = []
+    for _ in range(num_basic_block):
+        layers.append(basic_block(**kwarg))
+    return nn.Sequential(*layers)
+
+
+def act(act_type, inplace=True, neg_slope=0.2, n_prelu=1, beta=1.0):
+    """activation helper"""
+    act_type = act_type.lower()
+    if act_type == "relu":
+        layer = nn.ReLU(inplace)
+    elif act_type in ("leakyrelu", "lrelu"):
+        layer = nn.LeakyReLU(neg_slope, inplace)
+    elif act_type == "prelu":
+        layer = nn.PReLU(num_parameters=n_prelu, init=neg_slope)
+    elif act_type == "tanh":  # [-1, 1] range output
+        layer = nn.Tanh()
+    elif act_type == "sigmoid":  # [0, 1] range output
+        layer = nn.Sigmoid()
+    else:
+        raise NotImplementedError(f"activation layer [{act_type}] is not found")
+    return layer
+
+
+class Identity(nn.Module):
+    def __init__(self, *kwargs):
+        super(Identity, self).__init__()
+
+    def forward(self, x, *kwargs):
+        return x
+
+
+def norm(norm_type, nc):
+    """Return a normalization layer"""
+    norm_type = norm_type.lower()
+    if norm_type == "batch":
+        layer = nn.BatchNorm2d(nc, affine=True)
+    elif norm_type == "instance":
+        layer = nn.InstanceNorm2d(nc, affine=False)
+    elif norm_type == "none":
+
+        def norm_layer(x):
+            return Identity()
+    else:
+        raise NotImplementedError(f"normalization layer [{norm_type}] is not found")
+    return layer
+
+
+def pad(pad_type, padding):
+    """padding layer helper"""
+    pad_type = pad_type.lower()
+    if padding == 0:
+        return None
+    if pad_type == "reflect":
+        layer = nn.ReflectionPad2d(padding)
+    elif pad_type == "replicate":
+        layer = nn.ReplicationPad2d(padding)
+    elif pad_type == "zero":
+        layer = nn.ZeroPad2d(padding)
+    else:
+        raise NotImplementedError(f"padding layer [{pad_type}] is not implemented")
+    return layer
+
+
+def get_valid_padding(kernel_size, dilation):
+    kernel_size = kernel_size + (kernel_size - 1) * (dilation - 1)
+    padding = (kernel_size - 1) // 2
+    return padding
+
+
+class ShortcutBlock(nn.Module):
+    """Elementwise sum the output of a submodule to its input"""
+
+    def __init__(self, submodule):
+        super(ShortcutBlock, self).__init__()
+        self.sub = submodule
+
+    def forward(self, x):
+        output = x + self.sub(x)
+        return output
+
+    def __repr__(self):
+        return "Identity + \n|" + self.sub.__repr__().replace("\n", "\n|")
+
+
+def sequential(*args):
+    """Flatten Sequential. It unwraps nn.Sequential."""
+    if len(args) == 1:
+        if isinstance(args[0], OrderedDict):
+            raise NotImplementedError("sequential does not support OrderedDict input.")
+        return args[0]  # No sequential is needed.
+    modules = []
+    for module in args:
+        if isinstance(module, nn.Sequential):
+            for submodule in module.children():
+                modules.append(submodule)
+        elif isinstance(module, nn.Module):
+            modules.append(module)
+    return nn.Sequential(*modules)
+
+
+def conv_block(
+    in_nc,
+    out_nc,
+    kernel_size,
+    stride=1,
+    dilation=1,
+    groups=1,
+    bias=True,
+    pad_type="zero",
+    norm_type=None,
+    act_type="relu",
+    mode="CNA",
+    convtype="Conv2D",
+    spectral_norm=False,
+):
+    """Conv layer with padding, normalization, activation"""
+    assert mode in ["CNA", "NAC", "CNAC"], f"Wrong conv mode [{mode}]"
+    padding = get_valid_padding(kernel_size, dilation)
+    p = pad(pad_type, padding) if pad_type and pad_type != "zero" else None
+    padding = padding if pad_type == "zero" else 0
+
+    if convtype == "PartialConv2D":
+        # this is definitely not going to work, but PartialConv2d doesn't work anyway and this shuts up static analyzer
+        from torchvision.ops import PartialConv2d
+
+        c = PartialConv2d(
+            in_nc,
+            out_nc,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=bias,
+            groups=groups,
+        )
+    elif convtype == "DeformConv2D":
+        from torchvision.ops import DeformConv2d  # not tested
+
+        c = DeformConv2d(
+            in_nc,
+            out_nc,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=bias,
+            groups=groups,
+        )
+    elif convtype == "Conv3D":
+        c = nn.Conv3d(
+            in_nc,
+            out_nc,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=bias,
+            groups=groups,
+        )
+    else:
+        c = nn.Conv2d(
+            in_nc,
+            out_nc,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=bias,
+            groups=groups,
+        )
+
+    if spectral_norm:
+        c = nn.utils.spectral_norm(c)
+
+    a = act(act_type) if act_type else None
+    if "CNA" in mode:
+        n = norm(norm_type, out_nc) if norm_type else None
+        return sequential(p, c, n, a)
+    elif mode == "NAC":
+        if norm_type is None and act_type is not None:
+            a = act(act_type, inplace=False)
+        n = norm(norm_type, in_nc) if norm_type else None
+        return sequential(n, a, p, c)
+
+
+def load_models(
+    model_path: Path,
+    command_path: str = None,
+) -> list:
+    """
+    A one-and done loader to try finding the desired models in specified directories.
+
+    @param download_name: Specify to download from model_url immediately.
+    @param model_url: If no other models are found, this will be downloaded on upscale.
+    @param model_path: The location to store/find models in.
+    @param command_path: A command-line argument to search for models in first.
+    @param ext_filter: An optional list of filename extensions to filter by
+    @return: A list of paths containing the desired model(s)
+    """
+    output = []
+
+    try:
+        places = []
+        if command_path is not None and command_path != model_path:
+            pretrained_path = os.path.join(command_path, "experiments/pretrained_models")
+            if os.path.exists(pretrained_path):
+                print(f"Appending path: {pretrained_path}")
+                places.append(pretrained_path)
+            elif os.path.exists(command_path):
+                places.append(command_path)
+
+        places.append(model_path)
+
+    except Exception:
+        pass
+
+    return output
+
+
+def mod2normal(state_dict):
+    # this code is copied from https://github.com/victorca25/iNNfer
+    if "conv_first.weight" in state_dict:
+        crt_net = {}
+        items = list(state_dict)
+
+        crt_net["model.0.weight"] = state_dict["conv_first.weight"]
+        crt_net["model.0.bias"] = state_dict["conv_first.bias"]
+
+        for k in items.copy():
+            if "RDB" in k:
+                ori_k = k.replace("RRDB_trunk.", "model.1.sub.")
+                if ".weight" in k:
+                    ori_k = ori_k.replace(".weight", ".0.weight")
+                elif ".bias" in k:
+                    ori_k = ori_k.replace(".bias", ".0.bias")
+                crt_net[ori_k] = state_dict[k]
+                items.remove(k)
+
+        crt_net["model.1.sub.23.weight"] = state_dict["trunk_conv.weight"]
+        crt_net["model.1.sub.23.bias"] = state_dict["trunk_conv.bias"]
+        crt_net["model.3.weight"] = state_dict["upconv1.weight"]
+        crt_net["model.3.bias"] = state_dict["upconv1.bias"]
+        crt_net["model.6.weight"] = state_dict["upconv2.weight"]
+        crt_net["model.6.bias"] = state_dict["upconv2.bias"]
+        crt_net["model.8.weight"] = state_dict["HRconv.weight"]
+        crt_net["model.8.bias"] = state_dict["HRconv.bias"]
+        crt_net["model.10.weight"] = state_dict["conv_last.weight"]
+        crt_net["model.10.bias"] = state_dict["conv_last.bias"]
+        state_dict = crt_net
+    return state_dict
+
+
+def resrgan2normal(state_dict, nb=23):
+    # this code is copied from https://github.com/victorca25/iNNfer
+    if "conv_first.weight" in state_dict and "body.0.rdb1.conv1.weight" in state_dict:
+        re8x = 0
+        crt_net = {}
+        items = list(state_dict)
+
+        crt_net["model.0.weight"] = state_dict["conv_first.weight"]
+        crt_net["model.0.bias"] = state_dict["conv_first.bias"]
+
+        for k in items.copy():
+            if "rdb" in k:
+                ori_k = k.replace("body.", "model.1.sub.")
+                ori_k = ori_k.replace(".rdb", ".RDB")
+                if ".weight" in k:
+                    ori_k = ori_k.replace(".weight", ".0.weight")
+                elif ".bias" in k:
+                    ori_k = ori_k.replace(".bias", ".0.bias")
+                crt_net[ori_k] = state_dict[k]
+                items.remove(k)
+
+        crt_net[f"model.1.sub.{nb}.weight"] = state_dict["conv_body.weight"]
+        crt_net[f"model.1.sub.{nb}.bias"] = state_dict["conv_body.bias"]
+        crt_net["model.3.weight"] = state_dict["conv_up1.weight"]
+        crt_net["model.3.bias"] = state_dict["conv_up1.bias"]
+        crt_net["model.6.weight"] = state_dict["conv_up2.weight"]
+        crt_net["model.6.bias"] = state_dict["conv_up2.bias"]
+
+        if "conv_up3.weight" in state_dict:
+            # modification supporting: https://github.com/ai-forever/Real-ESRGAN/blob/main/RealESRGAN/rrdbnet_arch.py
+            re8x = 3
+            crt_net["model.9.weight"] = state_dict["conv_up3.weight"]
+            crt_net["model.9.bias"] = state_dict["conv_up3.bias"]
+
+        crt_net[f"model.{8+re8x}.weight"] = state_dict["conv_hr.weight"]
+        crt_net[f"model.{8+re8x}.bias"] = state_dict["conv_hr.bias"]
+        crt_net[f"model.{10+re8x}.weight"] = state_dict["conv_last.weight"]
+        crt_net[f"model.{10+re8x}.bias"] = state_dict["conv_last.bias"]
+
+        state_dict = crt_net
+    return state_dict
+
+
+def infer_params(state_dict):
+    # this code is copied from https://github.com/victorca25/iNNfer
+    scale2x = 0
+    scalemin = 6
+    n_uplayer = 0
+    plus = False
+
+    for block in list(state_dict):
+        parts = block.split(".")
+        n_parts = len(parts)
+        if n_parts == 5 and parts[2] == "sub":
+            nb = int(parts[3])
+        elif n_parts == 3:
+            part_num = int(parts[1])
+            if part_num > scalemin and parts[0] == "model" and parts[2] == "weight":
+                scale2x += 1
+            if part_num > n_uplayer:
+                n_uplayer = part_num
+                out_nc = state_dict[block].shape[0]
+        if not plus and "conv1x1" in block:
+            plus = True
+
+    nf = state_dict["model.0.weight"].shape[0]
+    in_nc = state_dict["model.0.weight"].shape[1]
+    out_nc = out_nc
+    scale = 2**scale2x
+
+    return in_nc, out_nc, nf, nb, plus, scale
+
+
+# https://github.com/philz1337x/clarity-upscaler/blob/e0cd797198d1e0e745400c04d8d1b98ae508c73b/modules/images.py#L64
+Grid = namedtuple("Grid", ["tiles", "tile_w", "tile_h", "image_w", "image_h", "overlap"])
+
+
+# https://github.com/philz1337x/clarity-upscaler/blob/e0cd797198d1e0e745400c04d8d1b98ae508c73b/modules/images.py#L67
+def split_grid(image, tile_w=512, tile_h=512, overlap=64):
+    w = image.width
+    h = image.height
+
+    non_overlap_width = tile_w - overlap
+    non_overlap_height = tile_h - overlap
+
+    cols = math.ceil((w - overlap) / non_overlap_width)
+    rows = math.ceil((h - overlap) / non_overlap_height)
+
+    dx = (w - tile_w) / (cols - 1) if cols > 1 else 0
+    dy = (h - tile_h) / (rows - 1) if rows > 1 else 0
+
+    grid = Grid([], tile_w, tile_h, w, h, overlap)
+    for row in range(rows):
+        row_images = []
+
+        y = int(row * dy)
+
+        if y + tile_h >= h:
+            y = h - tile_h
+
+        for col in range(cols):
+            x = int(col * dx)
+
+            if x + tile_w >= w:
+                x = w - tile_w
+
+            tile = image.crop((x, y, x + tile_w, y + tile_h))
+
+            row_images.append([x, tile_w, tile])
+
+        grid.tiles.append([y, tile_h, row_images])
+
+    return grid
+
+
+# https://github.com/philz1337x/clarity-upscaler/blob/e0cd797198d1e0e745400c04d8d1b98ae508c73b/modules/images.py#L104
+def combine_grid(grid):
+    def make_mask_image(r):
+        r = r * 255 / grid.overlap
+        r = r.astype(np.uint8)
+        return Image.fromarray(r, "L")
+
+    mask_w = make_mask_image(
+        np.arange(grid.overlap, dtype=np.float32).reshape((1, grid.overlap)).repeat(grid.tile_h, axis=0)
+    )
+    mask_h = make_mask_image(
+        np.arange(grid.overlap, dtype=np.float32).reshape((grid.overlap, 1)).repeat(grid.image_w, axis=1)
+    )
+
+    combined_image = Image.new("RGB", (grid.image_w, grid.image_h))
+    for y, h, row in grid.tiles:
+        combined_row = Image.new("RGB", (grid.image_w, h))
+        for x, w, tile in row:
+            if x == 0:
+                combined_row.paste(tile, (0, 0))
+                continue
+
+            combined_row.paste(tile.crop((0, 0, grid.overlap, h)), (x, 0), mask=mask_w)
+            combined_row.paste(tile.crop((grid.overlap, 0, w, h)), (x + grid.overlap, 0))
+
+        if y == 0:
+            combined_image.paste(combined_row, (0, 0))
+            continue
+
+        combined_image.paste(
+            combined_row.crop((0, 0, combined_row.width, grid.overlap)),
+            (0, y),
+            mask=mask_h,
+        )
+        combined_image.paste(
+            combined_row.crop((0, grid.overlap, combined_row.width, h)),
+            (0, y + grid.overlap),
+        )
+
+    return combined_image
+
+
+class UpscalerESRGAN:
+    def __init__(self, model_path: Path, device: torch.device, dtype: torch.dtype):
+        self.device = device
+        self.dtype = dtype
+        self.model_path = model_path
+        self.model = self.load_model(model_path)
+
+    def __call__(self, img: Image.Image) -> Image.Image:
+        return self.upscale_without_tiling(img)
+
+    def to(self, device: torch.device, dtype: torch.dtype):
+        self.device = device
+        self.dtype = dtype
+        self.model.to(device=device, dtype=dtype)
+
+    def load_model(self, path: Path) -> SRVGGNetCompact | RRDBNet:
+        filename = path
+        state_dict = torch.load(filename, weights_only=True, map_location=self.device)
+
+        if "params_ema" in state_dict:
+            state_dict = state_dict["params_ema"]
+        elif "params" in state_dict:
+            state_dict = state_dict["params"]
+            num_conv = 16 if "realesr-animevideov3" in filename else 32
+            model = SRVGGNetCompact(
+                num_in_ch=3,
+                num_out_ch=3,
+                num_feat=64,
+                num_conv=num_conv,
+                upscale=4,
+                act_type="prelu",
+            )
+            model.load_state_dict(state_dict)
+            model.eval()
+            return model
+
+        if "body.0.rdb1.conv1.weight" in state_dict and "conv_first.weight" in state_dict:
+            nb = 6 if "RealESRGAN_x4plus_anime_6B" in filename else 23
+            state_dict = resrgan2normal(state_dict, nb)
+        elif "conv_first.weight" in state_dict:
+            state_dict = mod2normal(state_dict)
+        elif "model.0.weight" not in state_dict:
+            raise Exception("The file is not a recognized ESRGAN model.")
+
+        in_nc, out_nc, nf, nb, plus, mscale = infer_params(state_dict)
+
+        model = RRDBNet(in_nc=in_nc, out_nc=out_nc, nf=nf, nb=nb, upscale=mscale, plus=plus)
+        model.load_state_dict(state_dict)
+        model.eval()
+
+        return model
+
+    def upscale_without_tiling(self, img: Image.Image) -> Image.Image:
+        img = np.array(img)
+        img = img[:, :, ::-1]
+        img = np.ascontiguousarray(np.transpose(img, (2, 0, 1))) / 255
+        img = torch.from_numpy(img).float()
+        img = img.unsqueeze(0).to(device=self.device, dtype=self.dtype)
+        with torch.no_grad():
+            output = self.model(img)
+        output = output.squeeze().float().cpu().clamp_(0, 1).numpy()
+        output = 255.0 * np.moveaxis(output, 0, 2)
+        output = output.astype(np.uint8)
+        output = output[:, :, ::-1]
+        return Image.fromarray(output, "RGB")
+
+    # https://github.com/philz1337x/clarity-upscaler/blob/e0cd797198d1e0e745400c04d8d1b98ae508c73b/modules/esrgan_model.py#L208
+    def upscale_with_tiling(self, img: Image.Image) -> Image.Image:
+        grid = split_grid(img)
+        newtiles = []
+        scale_factor = 1
+
+        for y, h, row in grid.tiles:
+            newrow = []
+            for tiledata in row:
+                x, w, tile = tiledata
+
+                output = self.upscale_without_tiling(tile)
+                scale_factor = output.width // tile.width
+
+                newrow.append([x * scale_factor, w * scale_factor, output])
+            newtiles.append([y * scale_factor, h * scale_factor, newrow])
+
+        newgrid = Grid(
+            newtiles,
+            grid.tile_w * scale_factor,
+            grid.tile_h * scale_factor,
+            grid.image_w * scale_factor,
+            grid.image_h * scale_factor,
+            grid.overlap * scale_factor,
+        )
+        output = combine_grid(newgrid)
+        return output