test all

_

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import torch
+from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
+import numpy as np
+import gc
+from diffusers.utils import export_to_video
+from PIL import Image
+
+pipe = DiffusionPipeline.from_pretrained("cerspense/zeroscope_v2_576w", torch_dtype=torch.float16)
+pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+pipe.enable_xformers_memory_efficient_attention()
+pipe.enable_vae_slicing()
+
+prompt = "spiderman running in the desert"
+video_frames = pipe(prompt, num_inference_steps=2, height=320, width=576, num_frames=24, output_type="latent".frames
+# video_path = export_to_video(video_frames, output_video_path="/home/patrick/videos/video_576_spiderman_24.mp4")
+
+pipe.to("cpu")
+del pipe
+gc.collect()
+torch.cuda.empty_cache()
+
+import ipdb; ipdb.set_trace()
+
+pipe = DiffusionPipeline.from_pretrained("cerspense/zeroscope_v2_XL", torch_dtype=torch.float16)
+pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+pipe.enable_xformers_memory_efficient_attention()
+pipe.enable_vae_slicing()
+
+video = [Image.fromarray(frame).resize((1024, 576)) for frame in video_frames]
+
+video_frames = pipe(prompt, video=video, num_inference_steps=2, strength=0.6).frames
+video_path = export_to_video(video_frames, output_video_path="/home/patrick/videos/video_1024_spiderman_24.mp4")

benchmark/sd_inpaint.py

@@ -4,6 +4,11 @@ import requests
 import torch
 from PIL import Image
 from io import BytesIO
+import os
+from pathlib import Path
+from huggingface_hub import HfApi
+
+api = HfApi()
 
 url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
 
@@ -24,13 +29,23 @@ run_compile = True  # Set True / False
 
 pipe = StableDiffusionInpaintPipeline.from_pretrained(path, torch_dtype=torch.float16)
 pipe = pipe.to("cuda:0")
-pipe.unet.to(memory_format=torch.channels_last)
 
-if run_compile:
-    print("Run torch compile")
-    pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)
+torch.manual_seed(33)
+
+prompt = "A cute dog"
+
+image = pipe(prompt=prompt, image=init_image, mask_image=mask_image, force_unmasked_unchanged=True).images[0]
+# image = pipe(prompt=prompt, image=init_image, mask_image=mask_image, force_unmasked_unchanged=False).images[0]
+
+file_name = "aa"
 
-prompt = "ghibli style, a fantasy landscape with castles"
+path = os.path.join(Path.home(), "images", f"{file_name}.png")
+image.save(path)
 
-for _ in range(3):
-    image = pipe(prompt=prompt, image=init_image, mask_image=mask_image).images[0]
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")

hey.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d14df8aac190ef2626cc2bd48c4428152a170029bc8322d190c1d2481d9c914
+size 8847488

run_local_img2img_xl.py

@@ -16,7 +16,8 @@ path = sys.argv[1]
 
 api = HfApi()
 start_time = time.time()
-pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(path)
+pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(path, torch_dtype=torch.float16)
+pipe.to("cuda")
 pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config)
 # pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
 # pipe = StableDiffusionImg2ImgXLPipeline.from_pretrained(path, torch_dtype=torch.float16, safety_checker=None
@@ -33,7 +34,7 @@ prompt = "An astronaut riding a green horse on Mars"
 # url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
 # 
 # response = requests.get(url)
-init_image = torch.from_numpy(np.load("/home/patrick/images/xl_latents.npy"))
+init_image = torch.from_numpy(np.load("/home/patrick/images/xl_latents.npy")).to("cuda")
 
 image = pipe(prompt=prompt, image=init_image).images[0]
 

run_local_xl.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 from diffusers import DiffusionPipeline, EulerDiscreteScheduler, StableDiffusionPipeline, KDPM2DiscreteScheduler, StableDiffusionImg2ImgPipeline, HeunDiscreteScheduler, KDPM2AncestralDiscreteScheduler, DDIMScheduler
 import time
+from pytorch_lightning import seed_everything
 import os
 from huggingface_hub import HfApi
 # from compel import Compel
@@ -11,33 +12,31 @@ import requests
 from PIL import Image
 from io import BytesIO
 
-path = sys.argv[1]
-
 api = HfApi()
 start_time = time.time()
-pipe = DiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16)
-pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config)
-# pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
-# pipe = StableDiffusionImg2ImgPipeline.from_pretrained(path, torch_dtype=torch.float16, safety_checker=None
 
-# compel = Compel(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder)
+use_refiner = bool(int(sys.argv[1]))
+
+pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-0.9", torch_dtype=torch.float16, variant="fp16", use_safetensors=True)
+pipe.to("cuda")
+# pipe.enable_model_cpu_offload()
 
+if use_refiner:
+    refiner = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-0.9", torch_dtype=torch.float16)
+    refiner.enable_model_cpu_offload()
 
-pipe = pipe.to("cuda")
+prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"
+seed_everything(0)
+image = pipe(prompt=prompt, output_type="latent" if use_refiner else "pil").images[0]
 
-prompt = "Elon Musk riding a green horse on Mars"
+if use_refiner:
+    image = refiner(prompt=prompt, image=image[None, :]).imagas[0]
 
 # pipe.unet.to(memory_format=torch.channels_last)
 # pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)
 # pipe(prompt=prompt, num_inference_steps=2).images[0]
 
-image = pipe(prompt=prompt, num_images_per_prompt=1, num_inference_steps=40, output_type="latent").images
-pipe.to("cpu")
-
-pipe = DiffusionPipeline.from_pretrained("/home/patrick/diffusers-sd-xl/stable-diffusion-xl-refiner-0.9", torch_dtype=torch.float16)
-pipe.to("cuda")
-
-image = pipe(prompt=prompt, image=image, strength=0.5).images[0]
+# image = pipe(prompt=prompt, num_images_per_prompt=1, num_inference_steps=40, output_type="latent").images
 
 file_name = f"aaa"
 path = os.path.join(Path.home(), "images", f"{file_name}.png")

run_video.py

@@ -1,12 +1,31 @@
 #!/usr/bin/env python3
 import torch
-from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
+from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler, TextToVideoSDPipeline
+import numpy as np
+import gc
 from diffusers.utils import export_to_video
+from PIL import Image
 
-pipe = DiffusionPipeline.from_pretrained("cerspense/zeroscope_v2_576w", torch_dtype=torch.float16)
+pipe = TextToVideoSDPipeline.from_pretrained("cerspense/zeroscope_v2_XL", torch_dtype=torch.float16)
 pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
 pipe.enable_model_cpu_offload()
+pipe.enable_vae_slicing()
 
-prompt = "Darth Vader is surfing on waves"
-video_frames = pipe(prompt, num_inference_steps=40, height=320, width=576, num_frames=24).frames
-video_path = export_to_video(video_frames, output_video_path="/home/patrick/videos/video_10.mp4")
+prompt = "spiderman running in the desert"
+video_frames = pipe(prompt, num_inference_steps=2, height=576, width=1024, num_frames=24).frames
+# video_path = export_to_video(video_frames, output_video_path="/home/patrick/videos/video_576_spiderman_24.mp4")
+pipe.to("cpu")
+import ipdb; ipdb.set_trace()
+
+del pipe
+gc.collect()
+torch.cuda.empty_cache()
+
+pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
+pipe.enable_vae_slicing()
+
+video = [Image.fromarray(frame).resize((1024, 576)) for frame in video_frames]
+
+video_frames = pipe(prompt, video=video, num_inference_steps=2, strength=0.6).frames
+video_path = export_to_video(video_frames, output_video_path="/home/patrick/videos/video_1024_spiderman_24.mp4")

~

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+from diffusers import DiffusionPipeline, EulerDiscreteScheduler, StableDiffusionPipeline, KDPM2DiscreteScheduler, StableDiffusionImg2ImgPipeline, HeunDiscreteScheduler, KDPM2AncestralDiscreteScheduler, DDIMScheduler, StableDiffusionXLImg2ImgPipeline
+import time
+import numpy as np
+import os
+from huggingface_hub import HfApi
+# from compel import Compel
+import torch
+import sys
+from pathlib import Path
+import requests
+from PIL import Image
+from io import BytesIO
+
+path = sys.argv[1]
+
+api = HfApi()
+start_time = time.time()
+pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained(path)
+pipe.to("cuda")
+pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config)
+# pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
+# pipe = StableDiffusionImg2ImgXLPipeline.from_pretrained(path, torch_dtype=torch.float16, safety_checker=None
+
+# compel = Compel(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder)
+
+
+prompt = "An astronaut riding a green horse on Mars"
+
+
+# pipe.unet.to(memory_format=torch.channels_last)
+# pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)
+# pipe(prompt=prompt, num_inference_steps=2).images[0]
+# url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"
+# 
+# response = requests.get(url)
+init_image = torch.from_numpy(np.load("/home/patrick/images/xl_latents.npy")).to("cuda")
+
+image = pipe(prompt=prompt, image=init_image).images[0]
+
+file_name = f"aaa"
+path = os.path.join(Path.home(), "images", f"{file_name}.png")
+image.save(path)
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=path.split("/")[-1],
+    repo_id="patrickvonplaten/images",
+    repo_type="dataset",
+)
+print(f"https://huggingface.co/datasets/patrickvonplaten/images/blob/main/{file_name}.png")