multimodalart HF staff commited on
Commit
0e80ee6
1 Parent(s): 49cebd1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +25 -9
app.py CHANGED
@@ -1,15 +1,18 @@
1
  import torch
2
  import spaces
3
  from diffusers import StableDiffusionPipeline, DDIMScheduler, AutoencoderKL
4
- from ip_adapter.ip_adapter_faceid import IPAdapterFaceID
5
  from huggingface_hub import hf_hub_download
6
  from insightface.app import FaceAnalysis
 
7
  import gradio as gr
8
  import cv2
9
 
10
  base_model_path = "SG161222/Realistic_Vision_V4.0_noVAE"
11
  vae_model_path = "stabilityai/sd-vae-ft-mse"
12
- ip_ckpt = hf_hub_download(repo_id='h94/IP-Adapter-FaceID', filename="ip-adapter-faceid_sd15.bin", repo_type="model")
 
 
13
 
14
  device = "cuda"
15
 
@@ -31,25 +34,37 @@ pipe = StableDiffusionPipeline.from_pretrained(
31
  )
32
 
33
  ip_model = IPAdapterFaceID(pipe, ip_ckpt, device)
 
34
 
35
  @spaces.GPU(enable_queue=True)
36
- def generate_image(images, prompt, negative_prompt, progress=gr.Progress(track_tqdm=True)):
37
  pipe.to(device)
38
  app = FaceAnalysis(name="buffalo_l", providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
39
  app.prepare(ctx_id=0, det_size=(640, 640))
40
-
41
  faceid_all_embeds = []
 
42
  for image in images:
43
  face = cv2.imread(image)
44
  faces = app.get(face)
45
  faceid_embed = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)
46
  faceid_all_embeds.append(faceid_embed)
47
-
 
 
 
48
  average_embedding = torch.mean(torch.stack(faceid_all_embeds, dim=0), dim=0)
49
 
50
- image = ip_model.generate(
51
- prompt=prompt, negative_prompt=negative_prompt, faceid_embeds=average_embedding, width=512, height=512, num_inference_steps=30
52
- )
 
 
 
 
 
 
 
53
  print(image)
54
  return image
55
  css = '''
@@ -66,7 +81,8 @@ demo = gr.Interface(
66
  gr.Textbox(label="Prompt",
67
  info="Try something like 'a photo of a man/woman/person'",
68
  placeholder="A photo of a [man/woman/person]..."),
69
- gr.Textbox(label="Negative Prompt", placeholder="low quality")
 
70
  ],
71
  outputs=[gr.Gallery(label="Generated Image")],
72
  title="IP-Adapter-FaceID demo",
 
1
  import torch
2
  import spaces
3
  from diffusers import StableDiffusionPipeline, DDIMScheduler, AutoencoderKL
4
+ from ip_adapter.ip_adapter_faceid import IPAdapterFaceID, IPAdapterFaceIDPlus
5
  from huggingface_hub import hf_hub_download
6
  from insightface.app import FaceAnalysis
7
+ from insightface.utils import face_align
8
  import gradio as gr
9
  import cv2
10
 
11
  base_model_path = "SG161222/Realistic_Vision_V4.0_noVAE"
12
  vae_model_path = "stabilityai/sd-vae-ft-mse"
13
+ image_encoder_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K"
14
+ ip_ckpt = hf_hub_download(repo_id="h94/IP-Adapter-FaceID", filename="ip-adapter-faceid_sd15.bin", repo_type="model")
15
+ ip_plus_ckpt = hf_hub_download(repo_id="h94/IP-Adapter-FaceID", filename="ip-adapter-faceid-plusv2_sd15.bin", repo_type="model")
16
 
17
  device = "cuda"
18
 
 
34
  )
35
 
36
  ip_model = IPAdapterFaceID(pipe, ip_ckpt, device)
37
+ ip_model_plus = IPAdapterFaceIDPlus(pipe, image_encoder_path, ip_plus_ckpt, device)
38
 
39
  @spaces.GPU(enable_queue=True)
40
+ def generate_image(images, prompt, negative_prompt, preserve_face_structure, progress=gr.Progress(track_tqdm=True)):
41
  pipe.to(device)
42
  app = FaceAnalysis(name="buffalo_l", providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
43
  app.prepare(ctx_id=0, det_size=(640, 640))
44
+
45
  faceid_all_embeds = []
46
+ first_iteration = True
47
  for image in images:
48
  face = cv2.imread(image)
49
  faces = app.get(face)
50
  faceid_embed = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)
51
  faceid_all_embeds.append(faceid_embed)
52
+ if(first_iteration):
53
+ face_image = face_align.norm_crop(face, landmark=faces[0].kps, image_size=224) # you can also segment the face
54
+ first_iteration = False
55
+
56
  average_embedding = torch.mean(torch.stack(faceid_all_embeds, dim=0), dim=0)
57
 
58
+ if(not preserve_face_structure):
59
+ image = ip_model.generate(
60
+ prompt=prompt, negative_prompt=negative_prompt, faceid_embeds=average_embedding,
61
+ width=512, height=512, num_inference_steps=30
62
+ )
63
+ else:
64
+ image = ip_model_plus.generate(
65
+ prompt=prompt, negative_prompt=negative_prompt, faceid_embeds=average_embedding,
66
+ face_image=face_image, shortcut=True, s_scale=1.5, width=512, height=512, num_inference_steps=30
67
+ )
68
  print(image)
69
  return image
70
  css = '''
 
81
  gr.Textbox(label="Prompt",
82
  info="Try something like 'a photo of a man/woman/person'",
83
  placeholder="A photo of a [man/woman/person]..."),
84
+ gr.Textbox(label="Negative Prompt", placeholder="low quality"),
85
+ gr.Checkbox(label="Preserve Face Structure", value=False),
86
  ],
87
  outputs=[gr.Gallery(label="Generated Image")],
88
  title="IP-Adapter-FaceID demo",