Upload folder using huggingface_hub

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "newdream-sdxl-20"]
+	path = models/newdream-sdxl-20
+	url = https://huggingface.co/stablediffusionapi/newdream-sdxl-20
+	branch = main

README.md

@@ -0,0 +1,12 @@
+# edge-maxxing-newdream-sdxl
+
+This holds the baseline for the SDXL Nvidia GeForce RTX 4090 contest, which can be forked freely and optimized
+
+Some recommendations are as follows:
+- Installing dependencies should be done in pyproject.toml, including git dependencies
+- Compiled models should be included directly in the repository(rather than compiling during loading), loading time matters far more than file sizes
+- Avoid changing `src/main.py`, as that includes mostly protocol logic. Most changes should be in `models` and `src/pipeline.py`
+- Change `requirements.txt` to add extra arguments to be used when installing the package
+
+For testing, you need a docker container with pytorch and ubuntu 22.04,
+you can download your listed dependencies with `pip install -r requirements.txt -e .`, and then running `start_inference`

models/newdream-sdxl-20/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

models/newdream-sdxl-20/.gitignore

@@ -0,0 +1,2 @@
+step_*
+epoch_*

models/newdream-sdxl-20/README.md

@@ -0,0 +1,66 @@
+---
+license: creativeml-openrail-m
+tags:
+  - stablediffusionapi.com
+  - stable-diffusion-api
+  - text-to-image
+  - ultra-realistic
+pinned: true
+---
+
+# NewDream-SDXL 2.0 API Inference
+
+![generated from stablediffusionapi.com](https://pub-3626123a908346a7a8be8d9295f44e26.r2.dev/generations/8478583971702167737.png)
+## Get API Key
+
+Get API key from [Stable Diffusion API](http://stablediffusionapi.com/), No Payment needed. 
+
+Replace Key in below code, change **model_id**  to "newdream-sdxl-20"
+
+Coding in PHP/Node/Java etc? Have a look at docs for more code examples: [View docs](https://stablediffusionapi.com/docs)
+
+Try model for free: [Generate Images](https://stablediffusionapi.com/models/newdream-sdxl-20)
+
+Model link: [View model](https://stablediffusionapi.com/models/newdream-sdxl-20)
+
+Credits: [View credits](https://civitai.com/?query=NewDream-SDXL%202.0)
+
+View all models: [View Models](https://stablediffusionapi.com/models)
+
+    import requests  
+    import json  
+      
+    url =  "https://stablediffusionapi.com/api/v4/dreambooth"  
+      
+    payload = json.dumps({  
+    "key":  "your_api_key",  
+    "model_id":  "newdream-sdxl-20",  
+    "prompt":  "ultra realistic close up portrait ((beautiful pale cyberpunk female with heavy black eyeliner)), blue eyes, shaved side haircut, hyper detail, cinematic lighting, magic neon, dark red city, Canon EOS R3, nikon, f/1.4, ISO 200, 1/160s, 8K, RAW, unedited, symmetrical balance, in-frame, 8K",  
+    "negative_prompt":  "painting, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, deformed, ugly, blurry, bad anatomy, bad proportions, extra limbs, cloned face, skinny, glitchy, double torso, extra arms, extra hands, mangled fingers, missing lips, ugly face, distorted face, extra legs, anime",  
+    "width":  "512",  
+    "height":  "512",  
+    "samples":  "1",  
+    "num_inference_steps":  "30",  
+    "safety_checker":  "no",  
+    "enhance_prompt":  "yes",  
+    "seed":  None,  
+    "guidance_scale":  7.5,  
+    "multi_lingual":  "no",  
+    "panorama":  "no",  
+    "self_attention":  "no",  
+    "upscale":  "no",  
+    "embeddings":  "embeddings_model_id",  
+    "lora":  "lora_model_id",  
+    "webhook":  None,  
+    "track_id":  None  
+    })  
+      
+    headers =  {  
+    'Content-Type':  'application/json'  
+    }  
+      
+    response = requests.request("POST", url, headers=headers, data=payload)  
+      
+    print(response.text)
+
+> Use this coupon code to get 25% off **DMGG0RBN** 

models/newdream-sdxl-20/model_index.json

@@ -0,0 +1,33 @@
+{
+  "_class_name": "StableDiffusionXLPipeline",
+  "_diffusers_version": "0.21.2",
+  "force_zeros_for_empty_prompt": true,
+  "scheduler": [
+    "diffusers",
+    "EulerDiscreteScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "text_encoder_2": [
+    "transformers",
+    "CLIPTextModelWithProjection"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "tokenizer_2": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "unet": [
+    "diffusers",
+    "UNet2DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ]
+}

models/newdream-sdxl-20/scheduler/scheduler_config.json

@@ -0,0 +1,18 @@
+{
+  "_class_name": "EulerDiscreteScheduler",
+  "_diffusers_version": "0.21.2",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "interpolation_type": "linear",
+  "num_train_timesteps": 1000,
+  "prediction_type": "epsilon",
+  "sample_max_value": 1.0,
+  "set_alpha_to_one": false,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "timestep_spacing": "leading",
+  "trained_betas": null,
+  "use_karras_sigmas": false
+}

models/newdream-sdxl-20/text_encoder/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "torch_dtype": "float16",
+  "transformers_version": "4.33.1",
+  "vocab_size": 49408
+}

models/newdream-sdxl-20/text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57f094c002b6f50986b68a714285e72a23c17d9e1b146b078a2219397c51e37a
+size 246144152

models/newdream-sdxl-20/text_encoder/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:192a06f4ef7ece4acb33fc3c717790ee37b1f5d85e48e0dcac54dfea93e584a2
+size 246185562

models/newdream-sdxl-20/text_encoder_2/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "CLIPTextModelWithProjection"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1280,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 20,
+  "num_hidden_layers": 32,
+  "pad_token_id": 1,
+  "projection_dim": 1280,
+  "torch_dtype": "float16",
+  "transformers_version": "4.33.1",
+  "vocab_size": 49408
+}

models/newdream-sdxl-20/text_encoder_2/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a0d55a94e8508c869f35163fb6fcf34e02ea1b614d9259b47f97c562cff9575
+size 1389382176

models/newdream-sdxl-20/text_encoder_2/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8841d5e4c05ce74941eb536ec3835f600cc82d36763fc5f30c69d09a886158c9
+size 1389490462

models/newdream-sdxl-20/tokenizer/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/newdream-sdxl-20/tokenizer/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/newdream-sdxl-20/tokenizer_2/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "!",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/newdream-sdxl-20/tokenizer_2/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "!",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

models/newdream-sdxl-20/unet/config.json

@@ -0,0 +1,71 @@
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.21.2",
+  "act_fn": "silu",
+  "addition_embed_type": "text_time",
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": 256,
+  "attention_head_dim": [
+    5,
+    10,
+    20
+  ],
+  "attention_type": "default",
+  "block_out_channels": [
+    320,
+    640,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "class_embeddings_concat": false,
+  "conv_in_kernel": 3,
+  "conv_out_kernel": 3,
+  "cross_attention_dim": 2048,
+  "cross_attention_norm": null,
+  "down_block_types": [
+    "DownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dropout": 0.0,
+  "dual_cross_attention": false,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_only_cross_attention": null,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 4,
+  "projection_class_embeddings_input_dim": 2816,
+  "resnet_out_scale_factor": 1.0,
+  "resnet_skip_time_act": false,
+  "resnet_time_scale_shift": "default",
+  "sample_size": 128,
+  "time_cond_proj_dim": null,
+  "time_embedding_act_fn": null,
+  "time_embedding_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "transformer_layers_per_block": [
+    1,
+    2,
+    10
+  ],
+  "up_block_types": [
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "UpBlock2D"
+  ],
+  "upcast_attention": false,
+  "use_linear_projection": true
+}

models/newdream-sdxl-20/unet/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56913b74b493462516ce770d525cca996feef7acd44cf0cba65de667a6c1d58c
+size 5135669022

models/newdream-sdxl-20/unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9715fb3c6d0d81f46fbe6be46ddce05d91806aaf041504a52a44e1ec9e4f660f
+size 5135149760

models/newdream-sdxl-20/vae/config.json

@@ -0,0 +1,31 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.21.2",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 1024,
+  "scaling_factor": 0.13025,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

models/newdream-sdxl-20/vae/diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c40ff3dc7adeb21dce76cd411d65828037efa0aa54432e3592418401cf8467
+size 167404866

models/newdream-sdxl-20/vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:915b909d0eeef5985462226b2c9950ca9da42b5a6ec8c296c2e73f6419ae465c
+size 167335342

pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["setuptools >= 61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "edge-maxxing-4090-newdream"
+description = "An edge-maxxing model submission for the 4090 newdream contest"
+requires-python = ">=3.10,<3.11"
+version = "1.0.0"
+dependencies = [
+    "diffusers==0.30.2",
+    "transformers==4.41.2",
+    "accelerate==0.31.0",
+    "omegaconf==2.3.0",
+    "tgate==1.0.0",
+    "DeepCache",
+    "PEFT",
+    "numpy==1.26.4",
+    "xformers==0.0.25.post1",
+    "triton==2.2.0",
+    "torch==2.2.2",
+    "torchvision==0.17.2",
+    "edge-maxxing-pipelines @ git+https://github.com/womboai/edge-maxxing#subdirectory=pipelines",
+]
+
+[project.scripts]
+start_inference = "main:main"

requirements.txt

@@ -0,0 +1 @@
+https://github.com/chengzeyi/stable-fast/releases/download/v1.0.5/stable_fast-1.0.5+torch222cu121-cp310-cp310-manylinux2014_x86_64.whl

src/main.py

@@ -0,0 +1,50 @@
+from io import BytesIO
+from multiprocessing.connection import Listener
+from os import chmod, remove
+from os.path import abspath, exists
+from pathlib import Path
+
+from PIL.JpegImagePlugin import JpegImageFile
+from pipelines.models import TextToImageRequest
+
+from pipeline import load_pipeline, infer
+
+SOCKET = abspath(Path(__file__).parent.parent / "inferences.sock")
+
+
+def main():
+    print(f"Loading pipeline")
+    pipeline = load_pipeline()
+
+    print(f"Pipeline loaded, creating socket at '{SOCKET}'")
+
+    if exists(SOCKET):
+        remove(SOCKET)
+
+    with Listener(SOCKET) as listener:
+        chmod(SOCKET, 0o777)
+
+        print(f"Awaiting connections")
+        with listener.accept() as connection:
+            print(f"Connected")
+
+            while True:
+                try:
+                    request = TextToImageRequest.model_validate_json(connection.recv_bytes().decode("utf-8"))
+                except EOFError:
+                    print(f"Inference socket exiting")
+
+                    return
+
+                image = infer(request, pipeline)
+
+                data = BytesIO()
+                image.save(data, format=JpegImageFile.format)
+
+                packet = data.getvalue()
+
+                connection.send_bytes(packet)
+
+
+if __name__ == '__main__':
+    main()

src/pipeline.py

@@ -0,0 +1,54 @@
+import torch
+from time import perf_counter
+from PIL.Image import Image
+from diffusers import StableDiffusionXLPipeline, DPMSolverMultistepScheduler, AutoencoderTiny, UNet2DConditionModel
+from pipelines.models import TextToImageRequest
+from torch import Generator
+from sfast.compilers.diffusion_pipeline_compiler import (compile,
+                                                         CompilationConfig)
+
+
+
+def load_pipeline() -> StableDiffusionXLPipeline:
+    pipeline = StableDiffusionXLPipeline.from_pretrained(
+        "./models/newdream-sdxl-20/",
+        torch_dtype=torch.float16,
+        use_safetensors=True,
+        variant="fp16",
+        local_files_only=True,)
+    pipeline.vae = AutoencoderTiny.from_pretrained("madebyollin/taesdxl", torch_dtype=torch.float16)
+    pipeline.scheduler = DPMSolverMultistepScheduler.from_config(
+    pipeline.scheduler.config)
+    pipeline.enable_freeu(s1=0.6, s2=0.4, b1=1.1, b2=1.2)
+    pipeline.to("cuda")
+
+    config = CompilationConfig.Default()
+
+    try:
+        import xformers
+        config.enable_xformers = True
+    except ImportError:
+        print('xformers not installed, skip')
+    try:
+        import triton
+        config.enable_triton = True
+    except ImportError:
+        print('Triton not installed, skip')
+
+    pipeline  = compile(pipeline, config)
+    for _ in range(4):
+        pipeline(prompt="", num_inference_steps=15,)
+
+    return pipeline
+
+def infer(request: TextToImageRequest, pipeline: StableDiffusionXLPipeline) -> Image:
+    generator = Generator(pipeline.device).manual_seed(request.seed) if request.seed else None
+
+    return pipeline(
+        prompt=request.prompt,
+        negative_prompt=request.negative_prompt,
+        width=request.width,
+        height=request.height,
+        generator=generator,
+        num_inference_steps=11,
+    ).images[0]