Update pipeline.py

pipeline.py

@@ -199,8 +199,11 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
     ):
         super().__init__()
         unet = UNetMotionModel.from_unet2d(unet, motion_adapter)
+        
+        # temp workaround to prevent ip adapter library from loading ip adapter on empty controlnet parameter
+        controlnet = controlnets
 
-        if controlnets is None:
+        if controlnet is None:
             self.register_modules(
                 vae=vae,
                 text_encoder=text_encoder,
@@ -218,7 +221,7 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
                 tokenizer=tokenizer,
                 unet=unet,
                 motion_adapter=motion_adapter,
-                controlnet=controlnets,
+                controlnet=controlnet,
                 scheduler=scheduler,
                 feature_extractor=feature_extractor,
                 image_encoder=image_encoder,
@@ -1117,8 +1120,8 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
                 returned, otherwise a `tuple` is returned where the first element is a list with the generated frames.
         """
         
-        if self.controlnets != None:
-            controlnets = self.controlnets._orig_mod if is_compiled_module(self.controlnets) else self.controlnets
+        if self.controlnet != None:
+            controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet
 
             # align format for control guidance
             control_end = control_guidance_end
@@ -1127,7 +1130,7 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
             elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list):
                 control_guidance_end = len(control_guidance_start) * [control_guidance_end]
             elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list):
-                mult = len(controlnets.nets) if isinstance(controlnets, MultiControlNetModel) else 1
+                mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1
                 control_guidance_start, control_guidance_end = (
                     mult * [control_guidance_start],
                     mult * [control_guidance_end],
@@ -1155,14 +1158,14 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
 
         device = self._execution_device
         
-        if self.controlnets != None:
-            if isinstance(controlnets, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):
-                controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnets.nets)
+        if self.controlnet != None:
+            if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):
+                controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets)
 
             global_pool_conditions = (
-                controlnets.config.global_pool_conditions
-                if isinstance(controlnets, ControlNetModel)
-                else controlnets.nets[0].config.global_pool_conditions
+                controlnet.config.global_pool_conditions
+                if isinstance(controlnet, ControlNetModel)
+                else controlnet.nets[0].config.global_pool_conditions
             )
             guess_mode = guess_mode or global_pool_conditions
             
@@ -1201,8 +1204,8 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
             if do_classifier_free_guidance:
                 image_embeds = torch.cat([negative_image_embeds, image_embeds])
 
-        if self.controlnets != None:
-            if isinstance(controlnets, ControlNetModel):
+        if self.controlnet != None:
+            if isinstance(controlnet, ControlNetModel):
                 # conditioning_frames = self.prepare_image(
                 #     image=conditioning_frames,
                 #     width=width,
@@ -1221,12 +1224,12 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
                     batch_size=batch_size * num_videos_per_prompt * num_frames,
                     num_images_per_prompt=num_videos_per_prompt,
                     device=device,
-                    dtype=controlnets.dtype,
+                    dtype=controlnet.dtype,
                     do_classifier_free_guidance=do_classifier_free_guidance,
                     guess_mode=guess_mode,
                 )
                                     
-            elif isinstance(controlnets, MultiControlNetModel):
+            elif isinstance(controlnet, MultiControlNetModel):
                 cond_prepared_frames = []
                 for frame_ in conditioning_frames:
                     # prepared_frame = self.prepare_image(
@@ -1248,7 +1251,7 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
                         batch_size=batch_size * num_videos_per_prompt * num_frames,
                         num_images_per_prompt=num_videos_per_prompt,
                         device=device,
-                        dtype=controlnets.dtype,
+                        dtype=controlnet.dtype,
                         do_classifier_free_guidance=do_classifier_free_guidance,
                         guess_mode=guess_mode,
                     )
@@ -1367,14 +1370,14 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
         added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None
         
         # 7.1 Create tensor stating which controlnets to keep
-        if self.controlnets != None:
+        if self.controlnet != None:
             controlnet_keep = []
             for i in range(len(timesteps)):
                 keeps = [
                     1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e)
                     for s, e in zip(control_guidance_start, control_guidance_end)
                 ]
-                controlnet_keep.append(keeps[0] if isinstance(controlnets, ControlNetModel) else keeps)
+                controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps)
 
         # divide the initial latents into context groups
         
@@ -1431,7 +1434,7 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
                     latent_model_input = torch.cat([current_context_latents] * 2) if do_classifier_free_guidance else current_context_latents
                     latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
 
-                    if self.controlnets != None and i < int(control_end*num_inference_steps):
+                    if self.controlnet != None and i < int(control_end*num_inference_steps):
                         
                         torch.cuda.synchronize()  # Synchronize GPU
                         control_start = time.time()
@@ -1465,7 +1468,7 @@ class AnimateDiffPipeline(DiffusionPipeline, TextualInversionLoaderMixin, IPAdap
                         )
 
     
-                        down_block_res_samples, mid_block_res_sample = self.controlnets(
+                        down_block_res_samples, mid_block_res_sample = self.controlnet(
                             control_model_input,
                             t,
                             encoder_hidden_states=controlnet_prompt_embeds,