up

control_net.py

@@ -1,34 +1,28 @@
 #!/usr/bin/env python3
 import torch
-import numpy as np
 import os
 from huggingface_hub import HfApi
 from pathlib import Path
-import cv2
-from PIL import Image
 from diffusers.utils import load_image
+from controlnet_aux import CannyDetector
 
 from diffusers import (
     ControlNetModel,
     StableDiffusionControlNetPipeline,
     UniPCMultistepScheduler,
 )
+import sys
+
+checkpoint = sys.argv[1]
 
 image = load_image(
-    "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png"
+    "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png"
 )
 
-image = np.array(image)
-
-low_threshold = 100
-high_threshold = 200
-
-image = cv2.Canny(image, low_threshold, high_threshold)
-image = image[:, :, None]
-image = np.concatenate([image, image, image], axis=2)
-canny_image = Image.fromarray(image)
+canny_detector = CannyDetector()
+canny_image = canny_detector(image, low_threshold=100, high_threshold=200)
 
-controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
+controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
 pipe = StableDiffusionControlNetPipeline.from_pretrained(
     "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
 )
@@ -36,8 +30,8 @@ pipe = StableDiffusionControlNetPipeline.from_pretrained(
 pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
 pipe.enable_model_cpu_offload()
 
-generator = torch.manual_seed(0)
-out_image = pipe("futuristic-looking woman", num_inference_steps=20, generator=generator, image=canny_image).images[0]
+generator = torch.manual_seed(33)
+out_image = pipe("a blue paradise bird in the jungle", num_inference_steps=20, generator=generator, image=canny_image).images[0]
 
 path = os.path.join(Path.home(), "images", "aa.png")
 out_image.save(path)

control_net_canny.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+import torch
+import os
+from huggingface_hub import HfApi
+from pathlib import Path
+from diffusers.utils import load_image
+from controlnet_aux import CannyDetector
+
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+import sys
+
+checkpoint = sys.argv[1]
+
+image = load_image(
+    "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png"
+)
+
+canny_detector = CannyDetector()
+canny_image = canny_detector(image, low_threshold=100, high_threshold=200)
+
+controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+)
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+
+generator = torch.manual_seed(33)
+out_image = pipe("a blue paradise bird in the jungle", num_inference_steps=20, generator=generator, image=canny_image).images[0]
+
+path = os.path.join(Path.home(), "images", "aa.png")
+out_image.save(path)
+
+api = HfApi()
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print("https://huggingface.co/datasets/patrickvonplaten/images/blob/main/aa.png")

control_net_depth.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import torch
+import os
+from huggingface_hub import HfApi
+from pathlib import Path
+from diffusers.utils import load_image
+from PIL import Image
+import numpy as np
+from transformers import pipeline
+
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+import sys
+
+checkpoint = sys.argv[1]
+
+image = load_image("https://huggingface.co/lllyasviel/sd-controlnet-depth/resolve/main/images/stormtrooper.png")
+
+prompt = "Stormtrooper's lecture in beautiful lecture hall"
+
+
+depth_estimator = pipeline('depth-estimation')
+image = depth_estimator(image)['depth']
+image = np.array(image)
+image = image[:, :, None]
+image = np.concatenate([image, image, image], axis=2)
+image = Image.fromarray(image)
+
+controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+)
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+
+generator = torch.manual_seed(0)
+out_image = pipe(prompt, num_inference_steps=40, generator=generator, image=image).images[0]
+
+path = os.path.join(Path.home(), "images", "aa.png")
+out_image.save(path)
+
+api = HfApi()
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print("https://huggingface.co/datasets/patrickvonplaten/images/blob/main/aa.png")

control_net_mlsd.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+import torch
+import os
+from huggingface_hub import HfApi
+from pathlib import Path
+from diffusers.utils import load_image
+from PIL import Image
+import numpy as np
+from controlnet_aux import MLSDdetector
+
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+import sys
+
+checkpoint = sys.argv[1]
+
+image = load_image("https://huggingface.co/lllyasviel/sd-controlnet-mlsd/resolve/main/images/room.png")
+
+prompt = "royal chamber with fancy bed"
+
+mlsd = MLSDdetector.from_pretrained('lllyasviel/ControlNet')
+image = mlsd(image)
+
+controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+)
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+
+generator = torch.manual_seed(0)
+out_image = pipe(prompt, num_inference_steps=30, generator=generator, image=image).images[0]
+
+path = os.path.join(Path.home(), "images", "aa.png")
+out_image.save(path)
+
+api = HfApi()
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print("https://huggingface.co/datasets/patrickvonplaten/images/blob/main/aa.png")

control_net_open_pose.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+import torch
+import os
+from huggingface_hub import HfApi
+from pathlib import Path
+from diffusers.utils import load_image
+from controlnet_aux import OpenposeDetector
+
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+import sys
+
+checkpoint = sys.argv[1]
+
+image = load_image("https://huggingface.co/lllyasviel/sd-controlnet-openpose/resolve/main/images/pose.png")
+prompt = "chef in the kitchen"
+
+
+openpose = OpenposeDetector.from_pretrained('lllyasviel/ControlNet')
+image = openpose(image)
+
+controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+)
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+
+generator = torch.manual_seed(33)
+out_image = pipe(prompt, num_inference_steps=20, generator=generator, image=image).images[0]
+
+path = os.path.join(Path.home(), "images", "aa.png")
+out_image.save(path)
+
+api = HfApi()
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print("https://huggingface.co/datasets/patrickvonplaten/images/blob/main/aa.png")

control_net_scribble.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+import torch
+import os
+from huggingface_hub import HfApi
+from pathlib import Path
+from diffusers.utils import load_image
+from PIL import Image
+import numpy as np
+from controlnet_aux import HEDdetector
+
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+import sys
+
+checkpoint = sys.argv[1]
+
+image = load_image("https://huggingface.co/lllyasviel/sd-controlnet-scribble/resolve/main/images/bag.png")
+
+prompt = "fancy Gucci bag on a table"
+
+hed = HEDdetector.from_pretrained('lllyasviel/ControlNet')
+image = hed(image, scribble=True)
+
+controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+)
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+
+generator = torch.manual_seed(33)
+out_image = pipe(prompt, num_inference_steps=30, generator=generator, image=image).images[0]
+
+path = os.path.join(Path.home(), "images", "aa.png")
+out_image.save(path)
+
+api = HfApi()
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print("https://huggingface.co/datasets/patrickvonplaten/images/blob/main/aa.png")

control_net_seg.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+import torch
+import os
+from huggingface_hub import HfApi
+from pathlib import Path
+from diffusers.utils import load_image
+from transformers import AutoImageProcessor, UperNetForSemanticSegmentation
+from PIL import Image
+import numpy as np
+
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    UniPCMultistepScheduler,
+)
+import sys
+
+image_processor = AutoImageProcessor.from_pretrained("openmmlab/upernet-convnext-small")
+image_segmentor = UperNetForSemanticSegmentation.from_pretrained("openmmlab/upernet-convnext-small")
+
+checkpoint = sys.argv[1]
+
+ada_palette = np.asarray([
+      [0, 0, 0],
+      [120, 120, 120],
+      [180, 120, 120],
+      [6, 230, 230],
+      [80, 50, 50],
+      [4, 200, 3],
+      [120, 120, 80],
+      [140, 140, 140],
+      [204, 5, 255],
+      [230, 230, 230],
+      [4, 250, 7],
+      [224, 5, 255],
+      [235, 255, 7],
+      [150, 5, 61],
+      [120, 120, 70],
+      [8, 255, 51],
+      [255, 6, 82],
+      [143, 255, 140],
+      [204, 255, 4],
+      [255, 51, 7],
+      [204, 70, 3],
+      [0, 102, 200],
+      [61, 230, 250],
+      [255, 6, 51],
+      [11, 102, 255],
+      [255, 7, 71],
+      [255, 9, 224],
+      [9, 7, 230],
+      [220, 220, 220],
+      [255, 9, 92],
+      [112, 9, 255],
+      [8, 255, 214],
+      [7, 255, 224],
+      [255, 184, 6],
+      [10, 255, 71],
+      [255, 41, 10],
+      [7, 255, 255],
+      [224, 255, 8],
+      [102, 8, 255],
+      [255, 61, 6],
+      [255, 194, 7],
+      [255, 122, 8],
+      [0, 255, 20],
+      [255, 8, 41],
+      [255, 5, 153],
+      [6, 51, 255],
+      [235, 12, 255],
+      [160, 150, 20],
+      [0, 163, 255],
+      [140, 140, 140],
+      [250, 10, 15],
+      [20, 255, 0],
+      [31, 255, 0],
+      [255, 31, 0],
+      [255, 224, 0],
+      [153, 255, 0],
+      [0, 0, 255],
+      [255, 71, 0],
+      [0, 235, 255],
+      [0, 173, 255],
+      [31, 0, 255],
+      [11, 200, 200],
+      [255, 82, 0],
+      [0, 255, 245],
+      [0, 61, 255],
+      [0, 255, 112],
+      [0, 255, 133],
+      [255, 0, 0],
+      [255, 163, 0],
+      [255, 102, 0],
+      [194, 255, 0],
+      [0, 143, 255],
+      [51, 255, 0],
+      [0, 82, 255],
+      [0, 255, 41],
+      [0, 255, 173],
+      [10, 0, 255],
+      [173, 255, 0],
+      [0, 255, 153],
+      [255, 92, 0],
+      [255, 0, 255],
+      [255, 0, 245],
+      [255, 0, 102],
+      [255, 173, 0],
+      [255, 0, 20],
+      [255, 184, 184],
+      [0, 31, 255],
+      [0, 255, 61],
+      [0, 71, 255],
+      [255, 0, 204],
+      [0, 255, 194],
+      [0, 255, 82],
+      [0, 10, 255],
+      [0, 112, 255],
+      [51, 0, 255],
+      [0, 194, 255],
+      [0, 122, 255],
+      [0, 255, 163],
+      [255, 153, 0],
+      [0, 255, 10],
+      [255, 112, 0],
+      [143, 255, 0],
+      [82, 0, 255],
+      [163, 255, 0],
+      [255, 235, 0],
+      [8, 184, 170],
+      [133, 0, 255],
+      [0, 255, 92],
+      [184, 0, 255],
+      [255, 0, 31],
+      [0, 184, 255],
+      [0, 214, 255],
+      [255, 0, 112],
+      [92, 255, 0],
+      [0, 224, 255],
+      [112, 224, 255],
+      [70, 184, 160],
+      [163, 0, 255],
+      [153, 0, 255],
+      [71, 255, 0],
+      [255, 0, 163],
+      [255, 204, 0],
+      [255, 0, 143],
+      [0, 255, 235],
+      [133, 255, 0],
+      [255, 0, 235],
+      [245, 0, 255],
+      [255, 0, 122],
+      [255, 245, 0],
+      [10, 190, 212],
+      [214, 255, 0],
+      [0, 204, 255],
+      [20, 0, 255],
+      [255, 255, 0],
+      [0, 153, 255],
+      [0, 41, 255],
+      [0, 255, 204],
+      [41, 0, 255],
+      [41, 255, 0],
+      [173, 0, 255],
+      [0, 245, 255],
+      [71, 0, 255],
+      [122, 0, 255],
+      [0, 255, 184],
+      [0, 92, 255],
+      [184, 255, 0],
+      [0, 133, 255],
+      [255, 214, 0],
+      [25, 194, 194],
+      [102, 255, 0],
+      [92, 0, 255],
+  ])
+
+image = load_image("https://huggingface.co/lllyasviel/sd-controlnet-seg/resolve/main/images/house.png").convert('RGB')
+
+prompt = "old house in stormy weather with rain and wind"
+
+pixel_values = image_processor(image, return_tensors="pt").pixel_values
+with torch.no_grad():
+  outputs = image_segmentor(pixel_values)
+seg = image_processor.post_process_semantic_segmentation(outputs, target_sizes=[image.size[::-1]])[0]
+color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8) # height, width, 3
+for label, color in enumerate(ada_palette):
+    color_seg[seg == label, :] = color
+color_seg = color_seg.astype(np.uint8)
+image = Image.fromarray(color_seg)
+
+
+controlnet = ControlNetModel.from_pretrained(checkpoint, torch_dtype=torch.float16)
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+)
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+
+generator = torch.manual_seed(0)
+out_image = pipe(prompt, num_inference_steps=30, generator=generator, image=image).images[0]
+
+path = os.path.join(Path.home(), "images", "aa.png")
+out_image.save(path)
+
+api = HfApi()
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print("https://huggingface.co/datasets/patrickvonplaten/images/blob/main/aa.png")

run_decomposed_if.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+from diffusers import IFBasePipeline, IFSuperResolutionPipeline, UNet2DConditionModel
+from transformers import T5EncoderModel, T5Tokenizer
+import torch
+import gc
+import os
+from pathlib import Path
+
+prompt = 'a photo of a kangaroo wearing an orange hoodie and blue sunglasses standing in front of the eiffel tower holding a sign that says "very deep learning"'
+
+model_id = "diffusers/if"
+model_id = "/home/patrick/if"
+
+# T5
+t5_tok = T5Tokenizer.from_pretrained(model_id, subfolder="tokenizer", torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+
+t5 = T5EncoderModel.from_pretrained(model_id, subfolder="text_encoder", torch_dtype=torch.float16, variant="fp16", low_cpu_mem_usage=True)
+t5.cuda()
+
+prompt = prompt.lower().strip()  # make sure everything is lower-cased
+with torch.no_grad():
+    inputs = t5_tok(prompt, max_length=77, return_tensors="pt", truncation=True, padding="max_length").to("cuda")
+    prompt_embeds = t5(**inputs).last_hidden_state
+
+with torch.no_grad():
+    inputs = t5_tok("", max_length=77, return_tensors="pt", truncation=True, padding="max_length").to("cuda")
+    neg_prompt_embeds = t5(**inputs).last_hidden_state
+
+del t5
+torch.cuda.empty_cache()
+gc.collect()
+
+generator = torch.Generator("cuda").manual_seed(0)
+
+# Stage 1
+pipe = IFBasePipeline.from_pretrained(model_id, text_encoder=None, torch_dtype=torch.float16, variant="fp16")
+pipe.to("cuda")
+
+image = pipe(prompt_embeds=prompt_embeds, negative_prompt_embeds=neg_prompt_embeds, output_type="pt", num_inference_steps=100, generator=generator).images
+
+# save_image
+pil_image = pipe.numpy_to_pil(pipe.decode_latents(image))[0]
+pil_image.save(os.path.join(Path.home(), "images", "if_I_0.png"))
+
+# offload
+del pipe
+torch.cuda.empty_cache()
+gc.collect()
+
+# Stage 2
+unet = UNet2DConditionModel.from_pretrained(model_id, subfolder="super_res_1_unet", torch_dtype=torch.float16)
+pipe = IFSuperResolutionPipeline.from_pretrained(model_id, unet=unet, text_encoder=None, torch_dtype=torch.float16, variant="fp16")
+pipe.to("cuda")
+
+image = pipe(image=image, prompt_embeds=prompt_embeds, negative_prompt_embeds=neg_prompt_embeds, num_inference_steps=50, noise_level=250, output_type="pt", generator=generator).images
+
+# save_image
+pil_image = pipe.numpy_to_pil(pipe.decode_latents(image))[0]
+pil_image.save(os.path.join(Path.home(), "images", "if_II_0.png"))
+
+# offload
+del pipe
+torch.cuda.empty_cache()
+gc.collect()
+
+# Stage 3
+unet = UNet2DConditionModel.from_pretrained(model_id, subfolder="super_res_2_unet", torch_dtype=torch.float16)
+pipe = IFSuperResolutionPipeline.from_pretrained(model_id, unet=unet, text_encoder=None, torch_dtype=torch.float16, variant="fp16")
+pipe.to("cuda")
+
+image = pipe(image=image, prompt_embeds=prompt_embeds, negative_prompt_embeds=neg_prompt_embeds, num_inference_steps=40, noise_level=0, output_type="pt", generator=generator).images
+
+# save image
+pil_image = pipe.numpy_to_pil(pipe.decode_latents(image))[0]
+pil_image.save(os.path.join(Path.home(), "images", "if_III_0.png"))
+
+# offload
+del pipe
+torch.cuda.empty_cache()
+gc.collect()

run_deepfloyd.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+from diffusers import DiffusionPipeline, DDPMScheduler
+import torch
+import time
+import os
+from pathlib import Path
+from huggingface_hub import HfApi
+import random
+import numpy as np
+from deepfloyd_if.modules import IFStageI, IFStageII, IFStageIII, T5Embedder
+import sys
+
+api = HfApi()
+start_time = time.time()
+seed = 0
+use_diffusers = bool(int(sys.argv[1]))
+
+t5_pos_embeds = torch.load("/home/patrick/tensors/embeds_orig.pt").to("cuda")
+t5_neg_embeds = torch.load("/home/patrick/tensors/neg_embeds.pt").to("cuda")
+
+def seed_everything(seed=None):
+    random.seed(seed)
+    os.environ['PYTHONHASHSEED'] = str(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = True
+    return seed
+
+if use_diffusers:
+    pipe = DiffusionPipeline.from_pretrained("/home/patrick/if-diff-ckpts/IF-I-IF-v1.0", torch_dtype=torch.float32, use_safetensors=True, text_encoder=None, safety_checker=None)
+    config = dict(pipe.scheduler.config)
+    config["timestep_spacing"] = "even_border"
+    pipe.scheduler = DDPMScheduler.from_config(config)
+    pipe.to("cuda")
+
+    with torch.no_grad():
+        # text_embeddings = t5.get_text_embeddings([prompt])
+        seed_everything(0)
+        out_image = pipe(prompt_embeds=t5_pos_embeds, negative_prompt_embeds=t5_neg_embeds, num_inference_steps=5).images[0]
+        out_image.save("/home/patrick/images/if_diff.png")
+else:
+    if_I = IFStageI(device="cuda", dir_or_name="/home/patrick/IF-I-IF-v1.0/", model_kwargs={"precision": "fp32"})
+    if_I_kwargs = {}
+    if_I_kwargs['negative_t5_embs'] = t5_neg_embeds
+    if_I_kwargs['seed'] = seed
+    if_I_kwargs['t5_embs'] = t5_pos_embeds
+    if_I_kwargs['aspect_ratio'] = "1:1"
+    if_I_kwargs['progress'] = True
+    if_I_kwargs['sample_timestep_respacing'] = '5'
+
+    seed_everything(0)
+    stageI_generations, _ = if_I.embeddings_to_image(**if_I_kwargs)
+
+    if_I.to_images(stageI_generations)[0].save("/home/patrick/images/if_ref.png")

run_if.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+#!/usr/bin/env python3
+from diffusers import DiffusionPipeline
+import torch
+import time
+import os
+from pathlib import Path
+from huggingface_hub import HfApi
+
+api = HfApi()
+start_time = time.time()
+
+pipe = DiffusionPipeline.from_pretrained("/home/patrick/if", torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+pipe.enable_model_cpu_offload()
+
+generator = torch.Generator("cuda").manual_seed(0)
+prompt = 'a photo of a kangaroo wearing an orange hoodie and blue sunglasses standing in front of the eiffel tower holding a sign that says "very deep learning"'
+
+image = pipe(prompt, generator=generator).images[0]
+
+path = os.path.join(Path.home(), "images", "if.png")
+image.save(path)
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/if.png")

run_if_orig.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+from deepfloyd_if.modules import IFStageI
+from time import time
+
+start_time = time()
+model = IFStageI(dir_or_name="/home/patrick/IF-I-IF-v1.0/", device="cuda")
+print("Time", time() - start_time)

run_if_stages.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+#!/usr/bin/env python3
+from diffusers import DiffusionPipeline
+import torch
+import time
+import os
+from pathlib import Path
+from huggingface_hub import HfApi
+
+api = HfApi()
+start_time = time.time()
+
+model_prefix = "diffusers"
+
+pipe = DiffusionPipeline.from_pretrained(f"{model_prefix}/IF-I-IF-v1.0", torch_dtype=torch.float16, safety_checker=None, variant="fp16", use_safetensors=True)
+pipe.enable_model_cpu_offload()
+
+super_res_1_pipe = DiffusionPipeline.from_pretrained(f"{model_prefix}/IF-II-L-v1.0", text_encoder=None, safety_checker=None, torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+super_res_1_pipe.enable_model_cpu_offload()
+
+super_res_2_pipe = DiffusionPipeline.from_pretrained(f"{model_prefix}/IF-III-L-v1.0", text_encoder=None, safety_checker=None, torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+super_res_2_pipe.enable_model_cpu_offload()
+
+prompt = 'a photo of a kangaroo wearing an orange hoodie and blue sunglasses standing in front of the eiffel tower holding a sign that says "very deep learning"'
+
+generator = torch.Generator("cuda").manual_seed(0)
+prompt_embeds, negative_embeds = pipe.encode_prompt(prompt)
+
+image = pipe(prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator, output_type="pt").images
+
+# save_image
+pil_image = pipe.numpy_to_pil(pipe.decode_latents(image))[0]
+pil_image.save(os.path.join(Path.home(), "images", "if_stage_I_0.png"))
+
+image = super_res_1_pipe(image=image, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator, output_type="pt", noise_level=250, num_inference_steps=50).images
+
+# save_image
+pil_image = pipe.numpy_to_pil(pipe.decode_latents(image))[0]
+pil_image.save(os.path.join(Path.home(), "images", "if_stage_II_0.png"))
+
+image = super_res_2_pipe(image=image, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, noise_level=0, num_inference_steps=40, generator=generator).images[0]
+
+# save_image
+image.save(os.path.join(Path.home(), "images", "if_stage_III_0.png"))