Repository: huggingface/transformers (Python)
Date: 2023-03-14 14:05:52+00:00
Task type: Bug Fix

Issue:
OneFormerProcessor、MaskFormerImageProcessor will cause errors if segmentation_maps only have elements 0 and 1
### System Info

transformers-4.26.0  do not have this bug
but transformers-4.27.0.dev0 has.

### Who can help?

_No response_

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [X] My own task or dataset (give details below)

### Reproduction

```python
from transformers import OneFormerProcessor, OneFormerForUniversalSegmentation, OneFormerImageProcessor, OneFormerConfig
from transformers import Mask2FormerImageProcessor, Mask2FormerForUniversalSegmentation
from PIL import Image
import requests
import torch
import numpy as np
import matplotlib

processor = OneFormerProcessor.from_pretrained("shi-labs/oneformer_ade20k_swin_tiny",num_text=134,do_reduce_labels=True,)

image_np=np.random.randint(0,255,(3,512,512))

#segmentation_maps only have elements 0 and 1
segmentation_maps = torch.randint(0, 2, (image_np.shape[1], image_np.shape[2]), dtype=torch.long)

inst2class={1: 4}

raw_inputs=processor.image_processor([image_np],
                   task_inputs=["panoptic"],
                   segmentation_maps=[segmentation_maps],
                   return_tensors="pt",
                   instance_id_to_semantic_id=inst2class,
                   do_reduce_labels=True,
                   ignore_index=None)

```
#ERROR
```
E:\condaenv\yaogan\lib\site-packages\transformers\models\oneformer\image_processing_oneformer.py:419: FutureWarning: The `reduce_labels` argument is deprecated and will be removed in v4.27. Please use `do_reduce_labels` instead.
  warnings.warn(
Traceback (most recent call last):
  File "E:\condaenv\yaogan\lib\site-packages\IPython\core\interactiveshell.py", line 3460, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-ed9733992fe8>", line 23, in <module>
    raw_inputs=processor.image_processor([image_np],
  File "E:\condaenv\yaogan\lib\site-packages\transformers\models\oneformer\image_processing_oneformer.py", line 524, in __call__
    return self.preprocess(images, task_inputs=task_inputs, segmentation_maps=segmentation_maps, **kwargs)
  File "E:\condaenv\yaogan\lib\site-packages\transformers\models\oneformer\image_processing_oneformer.py", line 708, in preprocess
    encoded_inputs = self.encode_inputs(
  File "E:\condaenv\yaogan\lib\site-packages\transformers\models\oneformer\image_processing_oneformer.py", line 962, in encode_inputs
    masks, classes = self.convert_segmentation_map_to_binary_masks(
  File "E:\condaenv\yaogan\lib\site-packages\transformers\models\oneformer\image_processing_oneformer.py", line 516, in convert_segmentation_map_to_binary_masks
    return convert_segmentation_map_to_binary_masks(
  File "E:\condaenv\yaogan\lib\site-packages\transformers\models\oneformer\image_processing_oneformer.py", line 288, in convert_segmentation_map_to_binary_masks
    class_id = instance_id_to_semantic_id[label + 1 if reduce_labels else label]
KeyError: 255
```
This bug is caused by a **resize** function of OneFormerProcessor, which convert segmentation_maps to PIL.Image and then convert to np.ndarray. After **resize**, segmentation_maps have elements 0 and 255, so the bug arise. 

### Expected behavior

fix this bug before release 4.27.0 as stable version
transformers-4.26.0  do not have this bug


Issue discussion:
cc @amyeroberts @alaradirik 

Solution patch:
diff --git a/src/transformers/image_transforms.py b/src/transformers/image_transforms.py
--- a/src/transformers/image_transforms.py
+++ b/src/transformers/image_transforms.py
@@ -156,12 +156,20 @@ def to_pil_image(
     # If there is a single channel, we squeeze it, as otherwise PIL can't handle it.
     image = np.squeeze(image, axis=-1) if image.shape[-1] == 1 else image
 
-    # PIL.Image can only store uint8 values, so we rescale the image to be between 0 and 255 if needed.
+    # PIL.Image can only store uint8 values so we rescale the image to be between 0 and 255 if needed.
     if do_rescale is None:
-        if np.all(0 <= image) and np.all(image <= 1):
-            do_rescale = True
-        elif np.allclose(image, image.astype(int)):
+        if image.dtype == np.uint8:
             do_rescale = False
+        elif np.allclose(image, image.astype(int)):
+            if np.all(0 <= image) and np.all(image <= 255):
+                do_rescale = False
+            else:
+                raise ValueError(
+                    "The image to be converted to a PIL image contains values outside the range [0, 255], "
+                    f"got [{image.min()}, {image.max()}] which cannot be converted to uint8."
+                )
+        elif np.all(0 <= image) and np.all(image <= 1):
+            do_rescale = True
         else:
             raise ValueError(
                 "The image to be converted to a PIL image contains values outside the range [0, 1], "


Test patch:
diff --git a/tests/test_image_transforms.py b/tests/test_image_transforms.py
--- a/tests/test_image_transforms.py
+++ b/tests/test_image_transforms.py
@@ -101,6 +101,27 @@ def test_to_pil_image_from_float(self, name, image_shape, dtype):
         with self.assertRaises(ValueError):
             to_pil_image(image)
 
+    @require_vision
+    def test_to_pil_image_from_mask(self):
+        # Make sure binary mask remains a binary mask
+        image = np.random.randint(0, 2, (3, 4, 5)).astype(np.uint8)
+        pil_image = to_pil_image(image)
+        self.assertIsInstance(pil_image, PIL.Image.Image)
+        self.assertEqual(pil_image.size, (5, 4))
+
+        np_img = np.asarray(pil_image)
+        self.assertTrue(np_img.min() == 0)
+        self.assertTrue(np_img.max() == 1)
+
+        image = np.random.randint(0, 2, (3, 4, 5)).astype(np.float32)
+        pil_image = to_pil_image(image)
+        self.assertIsInstance(pil_image, PIL.Image.Image)
+        self.assertEqual(pil_image.size, (5, 4))
+
+        np_img = np.asarray(pil_image)
+        self.assertTrue(np_img.min() == 0)
+        self.assertTrue(np_img.max() == 1)
+
     @require_tf
     def test_to_pil_image_from_tensorflow(self):
         # channels_first
@@ -222,7 +243,7 @@ def test_resize(self):
         self.assertIsInstance(resized_image, np.ndarray)
         self.assertEqual(resized_image.shape, (30, 40, 3))
 
-        # Check PIL.Image.Image is return if return_numpy=False
+        # Check PIL.Image.Image is returned if return_numpy=False
         resized_image = resize(image, (30, 40), return_numpy=False)
         self.assertIsInstance(resized_image, PIL.Image.Image)
         # PIL size is in (width, height) order