style

app.py

@@ -79,8 +79,8 @@ with gr.Blocks(css='style.css') as demo:
             create_demo_color(model.process_color)
         with gr.TabItem('Color + Sketch'):
             create_demo_color_sketch(model.process_color_sketch)
-        # with gr.TabItem('Style + Sketch'):
-        #     create_demo_style_sketch(model.process_style_sketch)
+        with gr.TabItem('Style + Sketch'):
+            create_demo_style_sketch(model.process_style_sketch)
         with gr.TabItem('Segmentation'):
             create_demo_seg(model.process_seg)
 demo.queue().launch(debug=True, server_name='0.0.0.0')

demo/model.py

@@ -177,8 +177,8 @@ class Model_all:
         # style part
         self.model_style = StyleAdapter(width=1024, context_dim=768, num_head=8, n_layes=3, num_token=8).to(device)
         self.model_style.load_state_dict(torch.load("models/t2iadapter_style_sd14v1.pth", map_location=device))
-        self.clip_processor = CLIPProcessor.from_pretrained('openai/clip-vit-large-patch14')
-        self.clip_vision_model = CLIPVisionModel.from_pretrained('openai/clip-vit-large-patch14').to(device)
+        self.clip_processor = CLIPProcessor.from_pretrained('models/clip/8d052a0f05efbaefbc9e8786ba291cfdf93e5bff')
+        self.clip_vision_model = CLIPVisionModel.from_pretrained('models/clip/8d052a0f05efbaefbc9e8786ba291cfdf93e5bff').to(device)
 
         device = 'cpu'
         ## mmpose
@@ -878,9 +878,8 @@ class Model_all:
         elif type_in == 'Image':
             from ldm.modules.structure_condition.openpose.api import OpenposeInference
             model = OpenposeInference()
-            keypose = model(im)
-            im_pose = keypose.copy()[:,:,::-1]
-            # keypose = img2tensor(keypose).unsqueeze(0) / 255.
+            keypose = model(im[:,:,::-1])
+            im_pose = keypose.copy()
 
         # extract condition features
         c = self.base_model.get_learned_conditioning([prompt + ', ' + pos_prompt])

ldm/models/diffusion/plms.py

@@ -117,7 +117,7 @@ class PLMSSampler(object):
                                                     features_adapter2=copy.deepcopy(features_adapter2),
                                                     mode = mode,
                                                     con_strength = con_strength, 
-                                                    style_feature=style_feature
+                                                    style_feature=style_feature#.clone()
                                                     )
         return samples, intermediates
 
@@ -152,6 +152,8 @@ class PLMSSampler(object):
             index = total_steps - i - 1
             ts = torch.full((b,), step, device=device, dtype=torch.long)
             ts_next = torch.full((b,), time_range[min(i + 1, len(time_range) - 1)], device=device, dtype=torch.long)
+            cond_in = cond
+            unconditional_conditioning_in = unconditional_conditioning
 
             if mask is not None :#and index>=10:
                 assert x0 is not None
@@ -170,20 +172,20 @@ class PLMSSampler(object):
                     features_adapter = features_adapter1
 
                 if index>25:
-                    cond = torch.cat([cond, style_feature], dim=1)
-                    unconditional_conditioning = torch.cat(
+                    cond_in = torch.cat([cond, style_feature.clone()], dim=1)
+                    unconditional_conditioning_in = torch.cat(
                         [unconditional_conditioning, unconditional_conditioning[:, -8:, :]], dim=1)
             elif mode == 'mul':
                 features_adapter = [a1i*0.5 + a2i for a1i, a2i in zip(features_adapter1, features_adapter2)]
             else:
                 features_adapter = features_adapter1
 
-            outs = self.p_sample_plms(img, cond, ts, index=index, use_original_steps=ddim_use_original_steps,
+            outs = self.p_sample_plms(img, cond_in, ts, index=index, use_original_steps=ddim_use_original_steps,
                                       quantize_denoised=quantize_denoised, temperature=temperature,
                                       noise_dropout=noise_dropout, score_corrector=score_corrector,
                                       corrector_kwargs=corrector_kwargs,
                                       unconditional_guidance_scale=unconditional_guidance_scale,
-                                      unconditional_conditioning=unconditional_conditioning,
+                                      unconditional_conditioning=unconditional_conditioning_in,
                                       old_eps=old_eps, t_next=ts_next, features_adapter=copy.deepcopy(features_adapter))
 
             img, pred_x0, e_t = outs