Apply inference on all images in the dataset
Opened by andysingal (closed)

### Describe the bug

```
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
Cell In[14], line 11
      9 for idx, example in enumerate(dataset['train']):
     10     image_path = example['image']
---> 11     mask_image = process_image(image_path)
     12     mask_image.save(f"mask_{idx}.png")

Cell In[14], line 4, in process_image(image_path)
      2 def process_image(image_path):
      3     print("Processing image:", image_path)
----> 4     result = inferencer(image_path)['predictions']
      5     mask = np.where(result == 12, 255, 0).astype('uint8')
      6     return Image.fromarray(mask)

File /usr/local/lib/python3.10/dist-packages/mmseg/apis/mmseg_inferencer.py:183, in MMSegInferencer.__call__(self, inputs, return_datasamples, batch_size, show, wait_time, out_dir, img_out_dir, pred_out_dir, **kwargs)
    180     pred_out_dir = ''
    181     img_out_dir = ''
--> 183 return super().__call__(
    184     inputs=inputs,
    185     return_datasamples=return_datasamples,
    186     batch_size=batch_size,
    187     show=show,
    188     wait_time=wait_time,
    189     img_out_dir=img_out_dir,
    190     pred_out_dir=pred_out_dir,
    191     **kwargs)

File /usr/local/lib/python3.10/dist-packages/mmengine/infer/infer.py:221, in BaseInferencer.__call__(self, inputs, return_datasamples, batch_size, **kwargs)
    218 inputs = self.preprocess(
    219     ori_inputs, batch_size=batch_size, **preprocess_kwargs)
    220 preds = []
--> 221 for data in (track(inputs, description='Inference')
    222              if self.show_progress else inputs):
    223     preds.extend(self.forward(data, **forward_kwargs))
    224 visualization = self.visualize(
    225     ori_inputs, preds,
    226     **visualize_kwargs)  # type: ignore  # noqa: E501

File /usr/local/lib/python3.10/dist-packages/rich/progress.py:168, in track(sequence, description, total, auto_refresh, console, transient, get_time, refresh_per_second, style, complete_style, finished_style, pulse_style, update_period, disable, show_speed)
    157 progress = Progress(
    158     *columns,
    159     auto_refresh=auto_refresh,
   (...)
    164     disable=disable,
    165 )
    167 with progress:
--> 168     yield from progress.track(
    169         sequence, total=total, description=description, update_period=update_period
    170     )

File /usr/local/lib/python3.10/dist-packages/rich/progress.py:1210, in Progress.track(self, sequence, total, task_id, description, update_period)
   1208 if self.live.auto_refresh:
   1209     with _TrackThread(self, task_id, update_period) as track_thread:
-> 1210         for value in sequence:
   1211             yield value
   1212             track_thread.completed += 1

File /usr/local/lib/python3.10/dist-packages/mmengine/infer/infer.py:291, in BaseInferencer.preprocess(self, inputs, batch_size, **kwargs)
    266 """Process the inputs into a model-feedable format.
    267 
    268 Customize your preprocess by overriding this method. Preprocess should
   (...)
    287     Any: Data processed by the ``pipeline`` and ``collate_fn``.
    288 """
    289 chunked_data = self._get_chunk_data(
    290     map(self.pipeline, inputs), batch_size)
--> 291 yield from map(self.collate_fn, chunked_data)

File /usr/local/lib/python3.10/dist-packages/mmengine/infer/infer.py:588, in BaseInferencer._get_chunk_data(self, inputs, chunk_size)
    586 chunk_data = []
    587 for _ in range(chunk_size):
--> 588     processed_data = next(inputs_iter)
    589     chunk_data.append(processed_data)
    590 yield chunk_data

File /usr/local/lib/python3.10/dist-packages/mmcv/transforms/base.py:12, in BaseTransform.__call__(self, results)
      9 def __call__(self,
     10              results: Dict) -> Optional[Union[Dict, Tuple[List, List]]]:
---> 12     return self.transform(results)

File /usr/local/lib/python3.10/dist-packages/mmcv/transforms/wrappers.py:88, in Compose.transform(self, results)
     79 """Call function to apply transforms sequentially.
     80 
     81 Args:
   (...)
     85     dict or None: Transformed results.
     86 """
     87 for t in self.transforms:
---> 88     results = t(results)  # type: ignore
     89     if results is None:
     90         return None

File /usr/local/lib/python3.10/dist-packages/mmcv/transforms/base.py:12, in BaseTransform.__call__(self, results)
      9 def __call__(self,
     10              results: Dict) -> Optional[Union[Dict, Tuple[List, List]]]:
---> 12     return self.transform(results)

File /usr/local/lib/python3.10/dist-packages/mmseg/datasets/transforms/loading.py:496, in InferencerLoader.transform(self, single_input)
    494     inputs = single_input
    495 else:
--> 496     raise NotImplementedError
    498 if 'img' in inputs:
    499     return self.from_ndarray(inputs)

NotImplementedError: 
````

### Steps to reproduce the bug

```
from datasets import load_dataset
dataset = load_dataset('Andyrasika/cat_kingdom')
dataset

from mmseg.apis import MMSegInferencer

checkpoint_name = 'segformer_mit-b5_8xb2-160k_ade20k-640x640'

inferencer = MMSegInferencer(model=checkpoint_name)

# Define a function to apply the code to each image in the dataset
def process_image(image_path):
    print("Processing image:", image_path)
    result = inferencer(image_path)['predictions']
    mask = np.where(result == 12, 255, 0).astype('uint8')
    return Image.fromarray(mask)

# Process and save masks for each image in the dataset
for idx, example in enumerate(dataset['train']):
    image_path = example['image']
    mask_image = process_image(image_path)
    mask_image.save(f"mask_{idx}.png")
```

### Expected behavior

create a separate column with masks in the dataset and further shows as a separate column in hub

### Environment info

jupyter notebook RTX 3090

Comments: From what I see, `MMSegInferencer` supports NumPy arrays, so replace the line `image_path = example['image']` with `image_path = np.array(example['image'])` to fix the issue (`example["image"]` is a `PIL.Image` object). 

> From what I see, `MMSegInferencer` supports NumPy arrays, so replace the line `image_path = example['image']` with `image_path = np.array(example['image'])` to fix the issue (`example["image"]` is a `PIL.Image` object).

Thanks @mariosasko for your reply...
i tried :
```
# Define a function to apply the code to each image in the dataset
def process_image(image_path):
    print("Processing image:", image_path)
    result = inferencer(image_path)['predictions']
    mask = np.where(result == 12, 255, 0).astype('uint8')
    return Image.fromarray(mask)

# Process and save masks for each image in the dataset
for idx, example in enumerate(dataset['train']):
    image_path = np.array(example['image'])
    mask_image = process_image(image_path)
    mask_image.save(f"mask_{idx}.png")
```
and got
```
Processing image: [[[202 165  87]
  [203 166  88]
  [207 168  91]
  ...
  [243 205 122]
  [244 202 120]
  [242 200 118]]

 [[202 165  87]
  [203 166  88]
  [207 168  91]
  ...
  [244 206 123]
  [245 203 121]
  [243 201 119]]

 [[203 164  87]
  [204 165  88]
  [207 168  91]
  ...
  [245 207 126]
  [246 204 122]
  [245 203 121]]

 ...

 [[154 123  56]
  [155 124  57]
  [158 125  56]
  ...
  [  3   3   1]
  [  3   3   1]
  [  3   3   1]]

 [[154 123  56]
  [154 123  56]
  [155 124  57]
  ...
  [  2   2   0]
  [  2   2   0]
  [  2   2   0]]

 [[152 121  54]
  [152 121  54]
  [153 122  55]
  ...
  [  2   2   0]
  [  2   2   0]
  [  2   2   0]]]
Inference ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   
Processing image: [[[ 39  44  40]
  [ 39  44  40]
  [ 39  43  44]
  ...
  [187 185 164]
  [208 204 175]
  [203 198 166]]

 [[ 42  47  43]
  [ 40  45  41]
  [ 40  44  45]
  ...
  [188 186 165]
  [202 198 169]
  [201 196 164]]

 [[ 41  46  42]
  [ 39  44  40]
  [ 40  44  45]
  ...
  [187 184 165]
  [197 193 166]
  [201 196 166]]

 ...

 [[ 29  27  30]
  [ 28  26  29]
  [ 25  23  26]
  ...
  [ 48  33  28]
  [ 44  31  25]
  [ 39  26  20]]

 [[ 34  29  33]
  [ 32  27  31]
  [ 29  24  28]
  ...
  [ 30  17  11]
  [ 36  23  15]
  [ 41  28  20]]

 [[ 35  30  34]
  [ 33  28  32]
  [ 28  23  27]
  ...
  [ 28  15   9]
  [ 41  28  20]
  [ 46  33  25]]]
Inference ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   
Processing image: [[[ 65  53  55]
  [ 65  53  55]
  [ 51  39  41]
  ...
  [133 127 111]
  [150 141 124]
  [133 124 107]]

 [[ 58  45  52]
  [ 61  48  55]
  [ 51  38  45]
  ...
  [148 141 123]
  [178 169 152]
  [144 135 118]]

 [[ 79  66  83]
  [ 73  60  77]
  [ 65  51  66]
  ...
  [140 131 114]
  [142 133 116]
  [147 136 118]]

 ...

 [[132 122 133]
  [ 95  85  94]
  [ 61  51  60]
  ...
  [ 39  28  42]
  [ 46  36  45]
  [ 25  16  21]]

 [[150 143 151]
  [114 107 115]
  [ 64  54  63]
  ...
  [ 47  35  47]
  [ 38  27  35]
  [140 129 133]]

 [[145 138 146]
  [115 108 116]
  [ 69  59  67]
  ...
  [ 31  19  31]
  [128 117 123]
  [196 185 189]]]
Inference ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━   
Processing image: [[[159 151 140]
  [171 163 152]
  [161 148 142]
  ...
  [198 184 171]
  [189 175 162]
  [183 169 156]]

 [[128 118 106]
  [138 128 116]
  [138 125 116]
  ...
  [200 186 173]
  [190 176 163]
  [187 173 160]]

 [[165 153 137]
  [170 158 142]
  [174 162 148]
  ...
  [200 187 171]
  [188 175 159]
  [182 169 153]]
```
However , when trying to add to:
```
from datasets import load_dataset
dataset = load_dataset('Andyrasika/cat_kingdom')
dataset
```
i did 
```
new_column = ["mask"] * len(dataset["train"])
new_column
dataset = dataset.add_column("/workspace/data", new_column)

print(dataset)
```
got error:
```
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[11], line 3
      1 new_column = ["mask"] * len(dataset["train"])
      2 new_column
----> 3 dataset = dataset.add_column("/workspace/data", new_column)
      5 print(dataset)

AttributeError: 'DatasetDict' object has no attribute 'add_column'
```

https://github.com/huggingface/datasets/issues/6246 resolved the `add_column` error, so I'm closing this issue :)