Repository: huggingface/diffusers

Issue:
🐛[`core`/`GLIGEN`]: TypeError when iterating over 0-d tensor with In-painting mode when EulerAncestralDiscreteScheduler is used
### Describe the bug

TypeError raised when trying to directly iterate over a 0-dimension tensor in the denoising stage of GLIGEN In-painting operation.

The error occurs when using diffusion noise schedulers that iterate over timesteps
(e.g. [EulerAnchestralDiscreteScheduler](https://huggingface.co/docs/diffusers/v0.21.0/en/api/schedulers/euler_ancestral), [KDPM2AncestralDiscreteScheduler](https://huggingface.co/docs/diffusers/v0.21.0/en/api/schedulers/dpm_discrete_ancestral)), during in-painting operation with the https://github.com/huggingface/diffusers/pull/4441 and https://github.com/huggingface/diffusers/pull/4777.

For further clarification, this operation of the add_noise function 🔽

[diffusers/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py](https://github.com/huggingface/diffusers/blob/ae2fc01a911d31eb95ee80fad3bcb25156dcb757/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py#L387-L388)

Lines 387 to 388 in [ae2fc01](https://github.com/huggingface/diffusers/commit/ae2fc01a911d31eb95ee80fad3bcb25156dcb757)

 step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] 
  
in the affected noise schedulers expects the timesteps to be a non-0 dim torch Tensor. However, in the affected pipelines, timesteps is 0-dimension.

### Reproduction

```python
import torch
from diffusers import StableDiffusionGLIGENPipeline
from diffusers.utils import load_image
import matplotlib.pyplot as plt

# Generate an image described by the prompt and
# insert objects described by text at the region defined by bounding boxes
pipe = StableDiffusionGLIGENPipeline.from_pretrained(
    "masterful/gligen-1-4-generation-text-box", variant="fp16", torch_dtype=torch.float16
)
pipe = pipe.to("cuda")

prompt = "a waterfall and a modern high speed train running through the tunnel in a beautiful forest with fall foliage"
boxes = [[0.1387, 0.2051, 0.4277, 0.7090], [0.4980, 0.4355, 0.8516, 0.7266]]
phrases = ["a waterfall", "a modern high speed train running through the tunnel"]

images = pipe(
    prompt=prompt,
    gligen_phrases=phrases,
    gligen_boxes=boxes,
    gligen_scheduled_sampling_beta=1,
    output_type="pil",
    num_inference_steps=50,
).images

plt.imshow(images[0])
```

### Logs

```shell
{
	"name": "TypeError",
	"message": "iteration over a 0-d tensor",
	"stack": "---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<path to python file> Cell 9 line 3
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=24'>25</a> pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=26'>27</a> # images = pipe(
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=27'>28</a> #     prompt=prompt,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=28'>29</a> #     image=input_image,
   (...)
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=31'>32</a> #     num_images_per_prompt=2,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=32'>33</a> # ).images
---> <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=33'>34</a> images = pipe(
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=34'>35</a>     prompt=prompt,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=35'>36</a>     gligen_phrases=phrases,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=36'>37</a>     gligen_boxes=boxes,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=37'>38</a>     gligen_scheduled_sampling_beta=1.0,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=38'>39</a>     gligen_inpaint_image = gligen_inpaint_image,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=39'>40</a>     output_type=\"np\",
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=40'>41</a>     num_inference_steps=50,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=41'>42</a>     num_images_per_prompt=1,
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=42'>43</a> ).images
     <a href='vscode-notebook-cell:<path to python file>#X11sdnNjb2RlLXJlbW90ZQ%3D%3D?line=44'>45</a> plt.imshow(images[0])

File ~/.conda/envs/conda_env/lib/python3.10/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File ~/defect_generation/diffusers_chu/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py:803, in StableDiffusionGLIGENPipeline.__call__(self, prompt, height, width, num_inference_steps, guidance_scale, gligen_scheduled_sampling_beta, gligen_phrases, gligen_boxes, gligen_inpaint_image, negative_prompt, num_images_per_prompt, eta, generator, latents, prompt_embeds, negative_prompt_embeds, output_type, return_dict, callback, callback_steps, cross_attention_kwargs, clip_skip)
    799     latents = torch.randn_like(latents[:, :4])
    801 if gligen_inpaint_image is not None:
    802     gligen_inpaint_latent_with_noise = (
--> 803         self.scheduler.add_noise(gligen_inpaint_latent, torch.randn_like(gligen_inpaint_latent), t)
    804         .expand(latents.shape[0], -1, -1, -1)
    805         .clone()
    806     )
    807     latents = gligen_inpaint_latent_with_noise * gligen_inpaint_mask + latents * (
    808         1 - gligen_inpaint_mask
    809     )
    811 # expand the latents if we are doing classifier free guidance

File ~/defect_generation/diffusers_chu/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py:387, in EulerAncestralDiscreteScheduler.add_noise(self, original_samples, noise, timesteps)
    384     schedule_timesteps = self.timesteps.to(original_samples.device)
    385     timesteps = timesteps.to(original_samples.device)
--> 387 step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps]
    389 sigma = sigmas[step_indices].flatten()
    390 while len(sigma.shape) < len(original_samples.shape):

File ~/.conda/envs/conda_env/lib/python3.10/site-packages/torch/_tensor.py:930, in Tensor.__iter__(self)
    920 def __iter__(self):
    921     # NB: we use 'imap' and not 'map' here, so that in Python 2 we get a
    922     # generator and don't eagerly perform all the indexes.  This could
   (...)
    927     # NB: We have intentionally skipped __torch_function__ dispatch here.
    928     # See gh-54457
    929     if self.dim() == 0:
--> 930         raise TypeError(\"iteration over a 0-d tensor\")
    931     if torch._C._get_tracing_state():
    932         warnings.warn(
    933             \"Iterating over a tensor might cause the trace to be incorrect. \"
    934             \"Passing a tensor of different shape won't change the number of \"
   (...)
    938             stacklevel=2,
    939         )

TypeError: iteration over a 0-d tensor" <=================ERROR!!!!!
}
```


### System Info

- `diffusers` version: 0.22.0.dev0
- Platform: Linux-5.15.0-60-generic-x86_64-with-glibc2.31
- Python version: 3.10.13
- PyTorch version (GPU?): 2.0.1 (True)
- Huggingface_hub version: 0.16.4
- Transformers version: 4.33.1
- Accelerate version: not installed
- xFormers version: not installed
- Using GPU in script?: Yes
- Using distributed or parallel set-up in script?: No

### Who can help?

cc: @sayakpaul , @nikhil-masterful, @tuanh123789


Discussion:

Resolving patch:
diff --git a/examples/custom_diffusion/train_custom_diffusion.py b/examples/custom_diffusion/train_custom_diffusion.py
--- a/examples/custom_diffusion/train_custom_diffusion.py
+++ b/examples/custom_diffusion/train_custom_diffusion.py
@@ -207,7 +207,7 @@ def __init__(
                     with open(concept["class_prompt"], "r") as f:
                         class_prompt = f.read().splitlines()
 
-                class_img_path = [(x, y) for (x, y) in zip(class_images_path, class_prompt)]
+                class_img_path = list(zip(class_images_path, class_prompt))
                 self.class_images_path.extend(class_img_path[:num_class_images])
 
         random.shuffle(self.instance_images_path)
diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py
--- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py
+++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py
@@ -803,7 +803,9 @@ def __call__(
 
                 if gligen_inpaint_image is not None:
                     gligen_inpaint_latent_with_noise = (
-                        self.scheduler.add_noise(gligen_inpaint_latent, torch.randn_like(gligen_inpaint_latent), t)
+                        self.scheduler.add_noise(
+                            gligen_inpaint_latent, torch.randn_like(gligen_inpaint_latent), torch.tensor([t])
+                        )
                         .expand(latents.shape[0], -1, -1, -1)
                         .clone()
                     )
diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen_text_image.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen_text_image.py
--- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen_text_image.py
+++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen_text_image.py
@@ -965,7 +965,9 @@ def __call__(
 
                 if gligen_inpaint_image is not None:
                     gligen_inpaint_latent_with_noise = (
-                        self.scheduler.add_noise(gligen_inpaint_latent, torch.randn_like(gligen_inpaint_latent), t)
+                        self.scheduler.add_noise(
+                            gligen_inpaint_latent, torch.randn_like(gligen_inpaint_latent), torch.tensor([t])
+                        )
                         .expand(latents.shape[0], -1, -1, -1)
                         .clone()
                     )


Test patch:
diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_gligen.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_gligen.py
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion_gligen.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_gligen.py
@@ -22,6 +22,7 @@
 from diffusers import (
     AutoencoderKL,
     DDIMScheduler,
+    EulerAncestralDiscreteScheduler,
     StableDiffusionGLIGENPipeline,
     UNet2DConditionModel,
 )
@@ -120,7 +121,7 @@ def get_dummy_inputs(self, device, seed=0):
         }
         return inputs
 
-    def test_gligen(self):
+    def test_stable_diffusion_gligen_default_case(self):
         device = "cpu"  # ensure determinism for the device-dependent torch.Generator
         components = self.get_dummy_components()
         sd_pipe = StableDiffusionGLIGENPipeline(**components)
@@ -136,6 +137,24 @@ def test_gligen(self):
 
         assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
 
+    def test_stable_diffusion_gligen_k_euler_ancestral(self):
+        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
+        components = self.get_dummy_components()
+        sd_pipe = StableDiffusionGLIGENPipeline(**components)
+        sd_pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(sd_pipe.scheduler.config)
+        sd_pipe = sd_pipe.to(device)
+        sd_pipe.set_progress_bar_config(disable=None)
+
+        inputs = self.get_dummy_inputs(device)
+        output = sd_pipe(**inputs)
+        image = output.images
+        image_slice = image[0, -3:, -3:, -1]
+
+        assert image.shape == (1, 64, 64, 3)
+        expected_slice = np.array([0.425, 0.494, 0.429, 0.469, 0.525, 0.417, 0.533, 0.5, 0.47])
+
+        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
+
     def test_attention_slicing_forward_pass(self):
         super().test_attention_slicing_forward_pass(expected_max_diff=3e-3)
 
diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_gligen_text_image.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_gligen_text_image.py
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion_gligen_text_image.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_gligen_text_image.py
@@ -29,6 +29,7 @@
 from diffusers import (
     AutoencoderKL,
     DDIMScheduler,
+    EulerAncestralDiscreteScheduler,
     StableDiffusionGLIGENTextImagePipeline,
     UNet2DConditionModel,
 )
@@ -150,7 +151,7 @@ def get_dummy_inputs(self, device, seed=0):
         }
         return inputs
 
-    def test_gligen(self):
+    def test_stable_diffusion_gligen_text_image_default_case(self):
         device = "cpu"  # ensure determinism for the device-dependent torch.Generator
         components = self.get_dummy_components()
         sd_pipe = StableDiffusionGLIGENTextImagePipeline(**components)
@@ -166,6 +167,24 @@ def test_gligen(self):
 
         assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
 
+    def test_stable_diffusion_gligen_k_euler_ancestral(self):
+        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
+        components = self.get_dummy_components()
+        sd_pipe = StableDiffusionGLIGENTextImagePipeline(**components)
+        sd_pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(sd_pipe.scheduler.config)
+        sd_pipe = sd_pipe.to(device)
+        sd_pipe.set_progress_bar_config(disable=None)
+
+        inputs = self.get_dummy_inputs(device)
+        image = sd_pipe(**inputs).images
+        image_slice = image[0, -3:, -3:, -1]
+
+        assert image.shape == (1, 64, 64, 3)
+
+        expected_slice = np.array([0.425, 0.494, 0.429, 0.469, 0.525, 0.417, 0.533, 0.5, 0.47])
+
+        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
+
     def test_attention_slicing_forward_pass(self):
         super().test_attention_slicing_forward_pass(expected_max_diff=3e-3)