add lightning

.lightning

@@ -0,0 +1 @@
+name: famous-carson-8575

.lightningignore

@@ -0,0 +1,32 @@
+__pycache__
+/ESRGAN/*
+/SwinIR/*
+/venv
+/tmp
+/GFPGANv1.3.pth
+/gfpgan/weights/*.pth
+/ui-config.json
+/outputs
+/log
+/webui.settings.bat
+/embeddings
+/styles.csv
+/params.txt
+/styles.csv.bak
+/interrogate
+/user.css
+/.idea
+notification.mp3
+/SwinIR
+/textual_inversion
+.vscode
+/extensions
+/test/stdout.txt
+/test/stderr.txt
+/cache.json
+.git
+*/chilloutmix_NiPrunedFp32Fix.safetensors
+*/vae-ft-mse-840000-ema-pruned.ckpt
+*/stLouisLuxuriousWheels_v1.safetensors
+*/taiwanDollLikeness_v10.safetensors
+*/koreanDollLikeness_v10.safetensors

README.md

@@ -1,7 +1,59 @@
 # Chill Watcher
 consider deploy on:
-- hugging-face inference point
+- huggingface inference point
 - replicate api
+- lightning.ai
+
+# platform comparison
+> all support autoscaling
+
+|platform|prediction speed|charges|deploy handiness|
+|-|-|-|-|
+|huggingface|fast:20s|high:$0.6/hr (without autoscaling)|easy:git push|
+|replicate|fast if used frequently: 30s, slow if needs initialization: 5min|low: $0.02 per generation|difficult: build image and upload|
+|lightning.ai|fast with app running: 20s, slow if idle: XXs|low: free $30 per month, XX per run|easy: one command|
+
+# platform deploy options
+## huggingface
+> [docs](https://huggingface.co/docs/inference-endpoints/guides/custom_handler)
+
+- requirements: use pip packages in `requirements.txt`
+- `init()` and `predict()` function: use `handler.py`, implement the `EndpointHandler` class
+- more: modify `handler.py` for requests and inference and explore more highly-customized features
+- deploy: git (lfs) push to huggingface repository(the whole directory including models and weights, etc.), and use inference endpoints to deploy. Click and deploy automaticly, very simple.
+- call api: use the url provide by inference endpoints after endpoint is ready(build, initialize and in a "running" state), make a post request to the url using request schema definied in the `handler.py`
+
+## replicate
+> [docs](https://replicate.com/docs/guides/push-a-model)
+
+- requirements: specify all requirements(pip packages, system packages, python version, cuda, etc.) in `cog.yaml`
+- `init()` and `predict()` function: use `predict.py`, implement the `Predictor` class
+- more: modify `predict.py`
+- deploy: 
+    1. get a linux GPU machine with 60GB disk space;
+    2. install [cog](https://replicate.com/docs/guides/push-a-model) and [docker](https://docs.docker.com/engine/install/ubuntu/#set-up-the-repository)
+    3. `git pull` the current repository from huggingface, including large model files
+    4. after `predict.py` and `cog.yaml` is correctly coded, run `cog login`, `cog push`, then cog will build a docker image locally and push the image to replicate. As the image could take 30GB or so disk space, it would cost a lot network bandwidth.
+- call api: if everything runs successfully and the docker image is pushed to replicate, you will see a web-ui and an API example directly in your replicate repository
+
+## lightning.ai
+> docs: [code](https://lightning.ai/docs/app/stable/levels/basic/real_lightning_component_implementations.html), [deploy](https://lightning.ai/docs/app/stable/workflows/run_app_on_cloud/)
+
+- requirements: 
+    - pip packages are listed in `requirements.txt`, note that some requirements are different from those in huggingface, and you need to modify some lines in `requirements.txt` according to the comment in the `requirements.txt`
+    - other pip packages, system packages and some big model weight files download commands, can be listed using a custom build config. Checkout `class CustomBuildConfig(BuildConfig)` in `app.py`. In a custom build config you can use many linux commands such as `wget` and `sudo apt-get update`. The custom build config will be executed on the `__init__()` of the `PythonServer` class
+- `init()` and `predict()` function: use `app.py`, implement the `PythonServer` class. Note: 
+    - some packages haven't been installed when the file is called(these packages may be installed when `__init__()` is called), so some import code should be in the function, not at the top of the file, or you may get import errors.
+    - you can't save your own value to `PythonServer.self` unless it's predifined in the variables, so don't assign any self-defined variables to `self`
+    - if you use the custom build config, you should implement `PythonServer`'s `__init()__` yourself, so don't forget to use the correct function signature
+- more: ...
+- deploy:
+    - `pip install lightning`
+    - prepare the directory on your local computer(no need to have a GPU)
+    - list big files in the `.lightningignore` file to avoid big file upload and save deploy time cost
+    - run `lightning run app app.py --cloud` in the local terminal, and it will upload the files in the directory to lightning cloud, and start deploying on the cloud
+    - check error logs on the web-ui, use `all logs`
+- call api: only if the app starts successfully, you can see a valid url in the `settings` page of the web-ui. Open that url, and you can see the api
 
 ### some stackoverflow:
 install docker:

app.py

@@ -0,0 +1,129 @@
+# inference handler for lightning ai
+
+import re
+import os
+import logging
+# import json
+from pydantic import BaseModel
+from typing import Any, Dict, Optional, TYPE_CHECKING
+from dataclasses import dataclass
+logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage())
+
+import lightning as L
+from lightning.app.components.serve import PythonServer, Text
+from lightning.app import BuildConfig
+
+
+class _DefaultInputData(BaseModel):
+    prompt: str
+
+class _DefaultOutputData(BaseModel):
+    prediction: str
+    parameters: str
+
+
+@dataclass
+class CustomBuildConfig(BuildConfig):
+    def build_commands(self):
+        dir_path = os.path.dirname(os.path.abspath(__file__))
+        model_path = os.path.join(dir_path, "models/Stable-diffusion")
+        model_url = "https://huggingface.co/Hardy01/chill_watcher/resolve/main/models/Stable-diffusion/chilloutmix_NiPrunedFp32Fix.safetensors"
+        download_cmd = "wget -P {} {}".format(str(model_path), model_url)
+        vae_url = "https://huggingface.co/Hardy01/chill_watcher/resolve/main/models/VAE/vae-ft-mse-840000-ema-pruned.ckpt"
+        vae_path = os.path.join(dir_path, "models/VAE")
+        down2 = "wget -P {} {}".format(str(vae_path), vae_url)
+        lora_url1 = "https://huggingface.co/Hardy01/chill_watcher/resolve/main/models/Lora/koreanDollLikeness_v10.safetensors"
+        lora_url2 = "https://huggingface.co/Hardy01/chill_watcher/resolve/main/models/Lora/taiwanDollLikeness_v10.safetensors"
+        lora_path = os.path.join(dir_path, "models/Lora")
+        down3 = "wget -P {} {}".format(str(lora_path), lora_url1)
+        down4 = "wget -P {} {}".format(str(lora_path), lora_url2)
+        # https://stackoverflow.com/questions/55313610/importerror-libgl-so-1-cannot-open-shared-object-file-no-such-file-or-directo
+        cmd1 = "pip3 install torch==1.13.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117"
+        cmd2 = "pip3 install torchvision==0.14.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117"
+        cmd_31 = "sudo apt-get update"
+        cmd3 = "sudo apt-get install libgl1-mesa-glx"
+        cmd4 = "sudo apt-get install libglib2.0-0"
+        return [download_cmd, down2, down3, down4, cmd1, cmd2, cmd_31, cmd3, cmd4]
+
+
+class PyTorchServer(PythonServer):
+    def __init__(
+        self,
+        input_type: type = _DefaultInputData,
+        output_type: type = _DefaultOutputData,
+        **kwargs: Any,
+        ):
+        super().__init__(input_type=input_type, output_type=output_type, **kwargs)
+
+        # Use the custom build config
+        self.cloud_build_config = CustomBuildConfig()
+    def setup(self):
+        # need to install dependancies first to import packages
+        import torch
+        # Truncate version number of nightly/local build of PyTorch to not cause exceptions with CodeFormer or Safetensors
+        if ".dev" in torch.__version__ or "+git" in torch.__version__:
+            torch.__long_version__ = torch.__version__
+            torch.__version__ = re.search(r'[\d.]+[\d]', torch.__version__).group(0)
+
+        from handler import initialize
+        initialize()
+
+    def predict(self, request):
+        from modules.api.api import encode_pil_to_base64
+        from modules import shared
+        from modules.processing import StableDiffusionProcessingTxt2Img, process_images
+        args = {
+            # todo: don't output png
+            "outpath_samples": "C:\\Users\\wolvz\\Desktop",
+            "prompt": "lora:koreanDollLikeness_v15:0.66, best quality, ultra high res, (photorealistic:1.4), 1girl, beige sweater, black choker, smile, laughing, bare shoulders, solo focus, ((full body), (brown hair:1), looking at viewer",
+            "negative_prompt": "paintings, sketches, (worst quality:2), (low quality:2), (normal quality:2), lowres, normal quality, ((monochrome)), ((grayscale)), skin spots, acnes, skin blemishes, age spot, glans, (ugly:1.331), (duplicate:1.331), (morbid:1.21), (mutilated:1.21), (tranny:1.331), mutated hands, (poorly drawn hands:1.331), blurry, 3hands,4fingers,3arms, bad anatomy, missing fingers, extra digit, fewer digits, cropped, jpeg artifacts,poorly drawn face,mutation,deformed",
+            "sampler_name": "DPM++ SDE Karras",
+            "steps": 20, # 25
+            "cfg_scale": 8,
+            "width": 512,
+            "height": 768,
+            "seed": -1,
+        }
+        print("&&&&&&&&&&&&&&&&&&&&&&&&",request)
+        if request.prompt:
+            prompt = request.prompt
+            print("get prompt from request: ", prompt)
+            args["prompt"] = prompt
+        p = StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)
+        processed = process_images(p)
+        single_image_b64 = encode_pil_to_base64(processed.images[0]).decode('utf-8')
+        return {
+            "prediction": single_image_b64,
+            "parameters": processed.images[0].info.get('parameters', ""),
+        }
+
+
+component = PyTorchServer(
+   cloud_compute=L.CloudCompute('gpu', disk_size=20, idle_timeout=30)
+)
+app = L.LightningApp(component)
+
+# class Flow(L.LightningFlow):
+#     # 1. Define the state
+#     def __init__(self):
+#         self.cloud_build_config = CustomBuildConfig()
+#         super().__init__()
+#         self.component = PyTorchServer(
+#             input_type=Text, output_type=Text, cloud_compute=L.CloudCompute('gpu', disk_size=20, idle_timeout=30)
+#         )
+
+#     # 2. Optional, but used to validate names
+#     def run(self):
+#         self.component.run()
+
+#     # 3. Method executed when a request is received.
+#     def handle_post(self, prompt: str):
+#         return f'The name {name} was registered'
+
+#     # 4. Defines this Component's Restful API. You can have several routes.
+#     def configure_api(self):
+#         return [Post(route="/name", method=self.handle_post)]
+
+
+# app = L.LightningApp(Flow())
+

handler.py

@@ -1,3 +1,4 @@
+# inference handler for huggingface
 import os
 import sys
 import time

play.py

@@ -0,0 +1,19 @@
+import requests
+import random
+import time
+import base64
+import hashlib
+import json
+
+def lightning():
+    start = int(time.time())
+    url = "https://wsoqr-01gwy9mc1gzh3b4ce9b708vp31.litng-ai-03.litng.ai/predict"
+    form = {
+        "prompt": "extremely detailed CG unity 8k wallpaper, masterpiece, best quality, ultra-detailed, best illustration, best shadow, photorealistic:1.4, 1 gorgeous girls,oversize pink_hoodie,under eiffel tower,grey_hair:1.1, collarbone,puffy breasts:1.5,full body shot,shiny eyes,enjoyable expression,evil smile,slim legs,narrow waist,detailed face, looking at viewer,looking back,gorgeous skin,short curly hair,kneeling,puffy ass up,climbing,lying,rosy pussy,nsfw,insert left_hand into pussy",
+    }
+    resp = requests.post(url, json=form)
+    print(resp.status_code, '\n', resp.content)
+    print("time cost(ms): ", int(time.time())*1e3-start*1e3)
+
+
+lightning()

predict.py

@@ -5,15 +5,9 @@ from cog import BasePredictor, Input, Path
 
 import os
 import sys
-import time
-import importlib
 import signal
 import re
 from typing import Dict, List, Any
-# from fastapi import FastAPI
-# from fastapi.middleware.cors import CORSMiddleware
-# from fastapi.middleware.gzip import GZipMiddleware
-from packaging import version
 
 import logging
 logging.getLogger("xformers").addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage())
@@ -29,6 +23,7 @@ if ".dev" in torch.__version__ or "+git" in torch.__version__:
     torch.__version__ = re.search(r'[\d.]+[\d]', torch.__version__).group(0)
 
 from modules import shared, devices, ui_tempdir
+from modules.api.api import encode_pil_to_base64
 import modules.codeformer_model as codeformer
 import modules.face_restoration
 import modules.gfpgan_model as gfpgan
@@ -51,38 +46,19 @@ from modules.shared import cmd_opts, opts
 import modules.hypernetworks.hypernetwork
 
 from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images
-import base64
-import io
-from fastapi import HTTPException
-from io import BytesIO
-import piexif
-import piexif.helper
-from PIL import PngImagePlugin,Image
 
 
 def initialize():
-    # check_versions()
-
-    # extensions.list_extensions()
-    # localization.list_localizations(cmd_opts.localizations_dir)
-
-    # if cmd_opts.ui_debug_mode:
-    #     shared.sd_upscalers = upscaler.UpscalerLanczos().scalers
-    #     modules.scripts.load_scripts()
-    #     return
-
     modelloader.cleanup_models()
     modules.sd_models.setup_model()
     codeformer.setup_model(cmd_opts.codeformer_models_path)
     gfpgan.setup_model(cmd_opts.gfpgan_models_path)
 
     modelloader.list_builtin_upscalers()
-    # modules.scripts.load_scripts()
     modelloader.load_upscalers()
 
     modules.sd_vae.refresh_vae_list()
 
-    # modules.textual_inversion.textual_inversion.list_textual_inversion_templates()
 
     try:
         modules.sd_models.load_model()
@@ -93,35 +69,11 @@ def initialize():
         exit(1)
 
     shared.opts.data["sd_model_checkpoint"] = shared.sd_model.sd_checkpoint_info.title
-
     shared.opts.onchange("sd_model_checkpoint", wrap_queued_call(lambda: modules.sd_models.reload_model_weights()))
     shared.opts.onchange("sd_vae", wrap_queued_call(lambda: modules.sd_vae.reload_vae_weights()), call=False)
     shared.opts.onchange("sd_vae_as_default", wrap_queued_call(lambda: modules.sd_vae.reload_vae_weights()), call=False)
     shared.opts.onchange("temp_dir", ui_tempdir.on_tmpdir_changed)
 
-    # shared.reload_hypernetworks()
-
-    # ui_extra_networks.intialize()
-    # ui_extra_networks.register_page(ui_extra_networks_textual_inversion.ExtraNetworksPageTextualInversion())
-    # ui_extra_networks.register_page(ui_extra_networks_hypernets.ExtraNetworksPageHypernetworks())
-    # ui_extra_networks.register_page(ui_extra_networks_checkpoints.ExtraNetworksPageCheckpoints())
-
-    # extra_networks.initialize()
-    # extra_networks.register_extra_network(extra_networks_hypernet.ExtraNetworkHypernet())
-
-    # if cmd_opts.tls_keyfile is not None and cmd_opts.tls_keyfile is not None:
-
-    #     try:
-    #         if not os.path.exists(cmd_opts.tls_keyfile):
-    #             print("Invalid path to TLS keyfile given")
-    #         if not os.path.exists(cmd_opts.tls_certfile):
-    #             print(f"Invalid path to TLS certfile: '{cmd_opts.tls_certfile}'")
-    #     except TypeError:
-    #         cmd_opts.tls_keyfile = cmd_opts.tls_certfile = None
-    #         print("TLS setup invalid, running webui without TLS")
-    #     else:
-    #         print("Running with TLS")
-
     # make the program just exit at ctrl+c without waiting for anything
     def sigint_handler(sig, frame):
         print(f'Interrupted with signal {sig} in {frame}')
@@ -129,104 +81,6 @@ def initialize():
 
     signal.signal(signal.SIGINT, sigint_handler)
 
-
-class EndpointHandler():
-    def __init__(self, path=""):
-        # Preload all the elements you are going to need at inference.
-        # pseudo:
-        # self.model= load_model(path)
-        initialize()
-        self.shared = shared
-
-    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
-        """
-        data args:
-            inputs (:obj: `str` | `PIL.Image` | `np.array`)
-            kwargs
-        Return:
-            A :obj:`list` | `dict`: will be serialized and returned
-        """
-        args = {
-            # todo: don't output png
-            "outpath_samples": "C:\\Users\\wolvz\\Desktop",
-            "prompt": "lora:koreanDollLikeness_v15:0.66, best quality, ultra high res, (photorealistic:1.4), 1girl, beige sweater, black choker, smile, laughing, bare shoulders, solo focus, ((full body), (brown hair:1), looking at viewer",
-            "negative_prompt": "paintings, sketches, (worst quality:2), (low quality:2), (normal quality:2), lowres, normal quality, ((monochrome)), ((grayscale)), skin spots, acnes, skin blemishes, age spot, glans, (ugly:1.331), (duplicate:1.331), (morbid:1.21), (mutilated:1.21), (tranny:1.331), mutated hands, (poorly drawn hands:1.331), blurry, 3hands,4fingers,3arms, bad anatomy, missing fingers, extra digit, fewer digits, cropped, jpeg artifacts,poorly drawn face,mutation,deformed",
-            "sampler_name": "DPM++ SDE Karras",
-            "steps": 20, # 25
-            "cfg_scale": 8,
-            "width": 512,
-            "height": 768,
-            "seed": -1,
-        }
-        if "prompt" in data.keys():
-            print("get prompt from request: ", data["prompt"])
-            args["prompt"] = data["prompt"]
-        p = StableDiffusionProcessingTxt2Img(sd_model=self.shared.sd_model, **args)
-        processed = process_images(p)
-        single_image_b64 = encode_pil_to_base64(processed.images[0]).decode('utf-8')
-        return {
-            "img_data": single_image_b64,
-            "parameters": processed.images[0].info.get('parameters', ""),
-        }
-
-
-def manual_hack():
-    initialize()
-    args = {
-        # todo: don't output res
-        "outpath_samples": "C:\\Users\\wolvz\\Desktop",
-        "prompt": "lora:koreanDollLikeness_v15:0.66, best quality, ultra high res, (photorealistic:1.4), 1girl, beige sweater, black choker, smile, laughing, bare shoulders, solo focus, ((full body), (brown hair:1), looking at viewer",
-        "negative_prompt": "paintings, sketches, (worst quality:2), (low quality:2), (normal quality:2), lowres, normal quality, ((monochrome)), ((grayscale)), skin spots, acnes, skin blemishes, age spot, glans",
-        "sampler_name": "DPM++ SDE Karras",
-        "steps": 20, # 25
-        "cfg_scale": 8,
-        "width": 512,
-        "height": 768,
-        "seed": -1,
-    }
-    p = StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)
-    processed = process_images(p)
-
-
-def decode_base64_to_image(encoding):
-    if encoding.startswith("data:image/"):
-        encoding = encoding.split(";")[1].split(",")[1]
-    try:
-        image = Image.open(BytesIO(base64.b64decode(encoding)))
-        return image
-    except Exception as err:
-        raise HTTPException(status_code=500, detail="Invalid encoded image")
-
-def encode_pil_to_base64(image):
-    with io.BytesIO() as output_bytes:
-
-        if opts.samples_format.lower() == 'png':
-            use_metadata = False
-            metadata = PngImagePlugin.PngInfo()
-            for key, value in image.info.items():
-                if isinstance(key, str) and isinstance(value, str):
-                    metadata.add_text(key, value)
-                    use_metadata = True
-            image.save(output_bytes, format="PNG", pnginfo=(metadata if use_metadata else None), quality=opts.jpeg_quality)
-
-        elif opts.samples_format.lower() in ("jpg", "jpeg", "webp"):
-            parameters = image.info.get('parameters', None)
-            exif_bytes = piexif.dump({
-                "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(parameters or "", encoding="unicode") }
-            })
-            if opts.samples_format.lower() in ("jpg", "jpeg"):
-                image.save(output_bytes, format="JPEG", exif = exif_bytes, quality=opts.jpeg_quality)
-            else:
-                image.save(output_bytes, format="WEBP", exif = exif_bytes, quality=opts.jpeg_quality)
-
-        else:
-            raise HTTPException(status_code=500, detail="Invalid image format")
-
-        bytes_data = output_bytes.getvalue()
-
-    return base64.b64encode(bytes_data)
-
-
 class Predictor(BasePredictor):
     def setup(self):
         """Load the model into memory to make running multiple predictions efficient"""