Add Diffusers weights

README.md

@@ -8,6 +8,7 @@ tags:
 - face
 - mediapipe
 license: "openrail"
+base_model: stabilityai/stable-diffusion-2-1-base
 datasets:
 - LAION-Face
 - LAION
@@ -110,6 +111,44 @@ model.load_state_dict(load_state_dict('./models/control_sd21_openpose.pth', loca
 
 The model has some limitations: while it is empirically better at tracking gaze and mouth poses than previous attempts, it may still ignore controls.  Adding details to the prompt like, "looking right" can abate bad behavior. 
 
+## 🧨 Diffusers
+
+It is recommended to use the checkpoint with [Stable Diffusion 2.1 - Base](stabilityai/stable-diffusion-2-1-base) as the checkpoint has been trained on it.
+Experimentally, the checkpoint can be used with other diffusion models such as dreamboothed stable diffusion.
+
+1. Let's install `diffusers` and related packages:
+```
+$ pip install diffusers transformers accelerate
+```
+
+2. Run code:
+```py
+from PIL import Image
+import numpy as np
+import torch
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
+from diffusers.utils import load_image
+
+image = load_image(
+    "https://huggingface.co/CrucibleAI/ControlNetMediaPipeFace/resolve/main/samples_laion_face_dataset/family_annotation.png"
+)
+
+controlnet = ControlNetModel.from_pretrained("CrucibleAI/ControlNetMediaPipeFace", torch_dtype=torch.float16, variant="fp16")
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+	"stabilityai/stable-diffusion-2-1-base", controlnet=controlnet, safety_checker=None, torch_dtype=torch.float16
+)
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+
+# Remove if you do not have xformers installed
+# see https://huggingface.co/docs/diffusers/v0.13.0/en/optimization/xformers#installing-xformers
+# for installation instructions
+pipe.enable_xformers_memory_efficient_attention()
+pipe.enable_model_cpu_offload()
+
+image = pipe("a happy family at a dentist advertisement", image=image, num_inference_steps=30).images[0]
+image.save('./images.png')
+```
+
 
 # License:
 

config.json

@@ -0,0 +1,47 @@
+{
+  "_class_name": "ControlNetModel",
+  "_diffusers_version": "0.15.0.dev0",
+  "_name_or_path": "/home/patrick_huggingface_co/temp_control",
+  "act_fn": "silu",
+  "attention_head_dim": [
+    5,
+    10,
+    20,
+    20
+  ],
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "class_embed_type": null,
+  "conditioning_embedding_out_channels": [
+    16,
+    32,
+    96,
+    256
+  ],
+  "controlnet_conditioning_channel_order": "rgb",
+  "cross_attention_dim": 1024,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_scale_factor": 1,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "upcast_attention": false,
+  "use_linear_projection": true
+}

diffusion_pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36dcd318d499df44b35432599a1b70f598e7bb42b479e4e67d4adf7b7e87e87d
+size 1457051321

diffusion_pytorch_model.fp16.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f70c38860e0d1fcd0f5ed38bc34e61c7337b9001bed57f7bff6eba6471406f0
+size 728596455

diffusion_pytorch_model.fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02b3a8e04154b4c3d11f5210217f0dbf3fac8612d62d015cd059f2b9fe4c3364
+size 728496846

diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a683e98e2427fd6242edc9af6620708f2f8fc84bfc049fafe549e350f8d42d73
+size 1456953564