GitHub issue: OSError: image file is truncated (1 bytes not processed) #28323
Author: andysingal (NONE)
State: closed | Created: 2024-01-04 02:15:13

### Describe the bug

```
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
Cell In[24], line 28
     23     return example
     25 # Filter the dataset
     26 # filtered_dataset = dataset.filter(contains_number)
     27 # Add the 'label' field in the dataset
---> 28 labeled_dataset = dataset.filter(contains_number).map(add_label)
     29 # View the structure of the updated dataset
     30 print(labeled_dataset)

File /usr/local/lib/python3.10/dist-packages/datasets/dataset_dict.py:975, in DatasetDict.filter(self, function, with_indices, input_columns, batched, batch_size, keep_in_memory, load_from_cache_file, cache_file_names, writer_batch_size, fn_kwargs, num_proc, desc)
    972 if cache_file_names is None:
    973     cache_file_names = {k: None for k in self}
    974 return DatasetDict(
--> 975     {
    976         k: dataset.filter(
    977             function=function,
    978             with_indices=with_indices,
    979             input_columns=input_columns,
    980             batched=batched,
    981             batch_size=batch_size,
    982             keep_in_memory=keep_in_memory,
    983             load_from_cache_file=load_from_cache_file,
    984             cache_file_name=cache_file_names[k],
    985             writer_batch_size=writer_batch_size,
    986             fn_kwargs=fn_kwargs,
    987             num_proc=num_proc,
    988             desc=desc,
    989         )
    990         for k, dataset in self.items()
    991     }
    992 )

File /usr/local/lib/python3.10/dist-packages/datasets/dataset_dict.py:976, in <dictcomp>(.0)
    972 if cache_file_names is None:
    973     cache_file_names = {k: None for k in self}
    974 return DatasetDict(
    975     {
--> 976         k: dataset.filter(
    977             function=function,
    978             with_indices=with_indices,
    979             input_columns=input_columns,
    980             batched=batched,
    981             batch_size=batch_size,
    982             keep_in_memory=keep_in_memory,
    983             load_from_cache_file=load_from_cache_file,
    984             cache_file_name=cache_file_names[k],
    985             writer_batch_size=writer_batch_size,
    986             fn_kwargs=fn_kwargs,
    987             num_proc=num_proc,
    988             desc=desc,
    989         )
    990         for k, dataset in self.items()
    991     }
    992 )

File /usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py:557, in transmit_format.<locals>.wrapper(*args, **kwargs)
    550 self_format = {
    551     "type": self._format_type,
    552     "format_kwargs": self._format_kwargs,
    553     "columns": self._format_columns,
    554     "output_all_columns": self._output_all_columns,
    555 }
    556 # apply actual function
--> 557 out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
    558 datasets: List["Dataset"] = list(out.values()) if isinstance(out, dict) else [out]
    559 # re-apply format to the output

File /usr/local/lib/python3.10/dist-packages/datasets/fingerprint.py:481, in fingerprint_transform.<locals>._fingerprint.<locals>.wrapper(*args, **kwargs)
    477             validate_fingerprint(kwargs[fingerprint_name])
    479 # Call actual function
--> 481 out = func(dataset, *args, **kwargs)
    483 # Update fingerprint of in-place transforms + update in-place history of transforms
    485 if inplace:  # update after calling func so that the fingerprint doesn't change if the function fails

File /usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py:3623, in Dataset.filter(self, function, with_indices, input_columns, batched, batch_size, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)
   3620 if len(self) == 0:
   3621     return self
-> 3623 indices = self.map(
   3624     function=partial(
   3625         get_indices_from_mask_function, function, batched, with_indices, input_columns, self._indices
   3626     ),
   3627     with_indices=True,
   3628     features=Features({"indices": Value("uint64")}),
   3629     batched=True,
   3630     batch_size=batch_size,
   3631     remove_columns=self.column_names,
   3632     keep_in_memory=keep_in_memory,
   3633     load_from_cache_file=load_from_cache_file,
   3634     cache_file_name=cache_file_name,
   3635     writer_batch_size=writer_batch_size,
   3636     fn_kwargs=fn_kwargs,
   3637     num_proc=num_proc,
   3638     suffix_template=suffix_template,
   3639     new_fingerprint=new_fingerprint,
   3640     input_columns=input_columns,
   3641     desc=desc or "Filter",
   3642 )
   3643 new_dataset = copy.deepcopy(self)
   3644 new_dataset._indices = indices.data

File /usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py:592, in transmit_tasks.<locals>.wrapper(*args, **kwargs)
    590     self: "Dataset" = kwargs.pop("self")
    591 # apply actual function
--> 592 out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
    593 datasets: List["Dataset"] = list(out.values()) if isinstance(out, dict) else [out]
    594 for dataset in datasets:
    595     # Remove task templates if a column mapping of the template is no longer valid

File /usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py:557, in transmit_format.<locals>.wrapper(*args, **kwargs)
    550 self_format = {
    551     "type": self._format_type,
    552     "format_kwargs": self._format_kwargs,
    553     "columns": self._format_columns,
    554     "output_all_columns": self._output_all_columns,
    555 }
    556 # apply actual function
--> 557 out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
    558 datasets: List["Dataset"] = list(out.values()) if isinstance(out, dict) else [out]
    559 # re-apply format to the output

File /usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py:3093, in Dataset.map(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)
   3087 if transformed_dataset is None:
   3088     with hf_tqdm(
   3089         unit=" examples",
   3090         total=pbar_total,
   3091         desc=desc or "Map",
   3092     ) as pbar:
-> 3093         for rank, done, content in Dataset._map_single(**dataset_kwargs):
   3094             if done:
   3095                 shards_done += 1

File /usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py:3470, in Dataset._map_single(shard, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset)
   3466 indices = list(
   3467     range(*(slice(i, i + batch_size).indices(shard.num_rows)))
   3468 )  # Something simpler?
   3469 try:
-> 3470     batch = apply_function_on_filtered_inputs(
   3471         batch,
   3472         indices,
   3473         check_same_num_examples=len(shard.list_indexes()) > 0,
   3474         offset=offset,
   3475     )
   3476 except NumExamplesMismatchError:
   3477     raise DatasetTransformationNotAllowedError(
   3478         "Using `.map` in batched mode on a dataset with attached indexes is allowed only if it doesn't create or remove existing examples. You can first run `.drop_index() to remove your index and then re-add it."
   3479     ) from None

File /usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py:3349, in Dataset._map_single.<locals>.apply_function_on_filtered_inputs(pa_inputs, indices, check_same_num_examples, offset)
   3347 if with_rank:
   3348     additional_args += (rank,)
-> 3349 processed_inputs = function(*fn_args, *additional_args, **fn_kwargs)
   3350 if isinstance(processed_inputs, LazyDict):
   3351     processed_inputs = {
   3352         k: v for k, v in processed_inputs.data.items() if k not in processed_inputs.keys_to_format
   3353     }

File /usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py:6212, in get_indices_from_mask_function(function, batched, with_indices, input_columns, indices_mapping, *args, **fn_kwargs)
   6209 if input_columns is None:
   6210     # inputs only contains a batch of examples
   6211     batch: dict = inputs[0]
-> 6212     num_examples = len(batch[next(iter(batch.keys()))])
   6213     for i in range(num_examples):
   6214         example = {key: batch[key][i] for key in batch}

File /usr/local/lib/python3.10/dist-packages/datasets/formatting/formatting.py:272, in LazyDict.__getitem__(self, key)
    270 value = self.data[key]
    271 if key in self.keys_to_format:
--> 272     value = self.format(key)
    273     self.data[key] = value
    274     self.keys_to_format.remove(key)

File /usr/local/lib/python3.10/dist-packages/datasets/formatting/formatting.py:375, in LazyBatch.format(self, key)
    374 def format(self, key):
--> 375     return self.formatter.format_column(self.pa_table.select([key]))

File /usr/local/lib/python3.10/dist-packages/datasets/formatting/formatting.py:442, in PythonFormatter.format_column(self, pa_table)
    440 def format_column(self, pa_table: pa.Table) -> list:
    441     column = self.python_arrow_extractor().extract_column(pa_table)
--> 442     column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
    443     return column

File /usr/local/lib/python3.10/dist-packages/datasets/formatting/formatting.py:218, in PythonFeaturesDecoder.decode_column(self, column, column_name)
    217 def decode_column(self, column: list, column_name: str) -> list:
--> 218     return self.features.decode_column(column, column_name) if self.features else column

File /usr/local/lib/python3.10/dist-packages/datasets/features/features.py:1951, in Features.decode_column(self, column, column_name)
   1938 def decode_column(self, column: list, column_name: str):
   1939     """Decode column with custom feature decoding.
   1940 
   1941     Args:
   (...)
   1948         `list[Any]`
   1949     """
   1950     return (
-> 1951         [decode_nested_example(self[column_name], value) if value is not None else None for value in column]
   1952         if self._column_requires_decoding[column_name]
   1953         else column
   1954     )

File /usr/local/lib/python3.10/dist-packages/datasets/features/features.py:1951, in <listcomp>(.0)
   1938 def decode_column(self, column: list, column_name: str):
   1939     """Decode column with custom feature decoding.
   1940 
   1941     Args:
   (...)
   1948         `list[Any]`
   1949     """
   1950     return (
-> 1951         [decode_nested_example(self[column_name], value) if value is not None else None for value in column]
   1952         if self._column_requires_decoding[column_name]
   1953         else column
   1954     )

File /usr/local/lib/python3.10/dist-packages/datasets/features/features.py:1339, in decode_nested_example(schema, obj, token_per_repo_id)
   1336 elif isinstance(schema, (Audio, Image)):
   1337     # we pass the token to read and decode files from private repositories in streaming mode
   1338     if obj is not None and schema.decode:
-> 1339         return schema.decode_example(obj, token_per_repo_id=token_per_repo_id)
   1340 return obj

File /usr/local/lib/python3.10/dist-packages/datasets/features/image.py:185, in Image.decode_example(self, value, token_per_repo_id)
    183 else:
    184     image = PIL.Image.open(BytesIO(bytes_))
--> 185 image.load()  # to avoid "Too many open files" errors
    186 return image

File /usr/local/lib/python3.10/dist-packages/PIL/ImageFile.py:254, in ImageFile.load(self)
    252         break
    253     else:
--> 254         raise OSError(
    255             "image file is truncated "
    256             f"({len(b)} bytes not processed)"
    257         )
    259 b = b + s
    260 n, err_code = decoder.decode(b)

OSError: image file is truncated (1 bytes not processed)
```

### Steps to reproduce the bug

```
from datasets import load_dataset

dataset = load_dataset("mehul7/captioned_military_aircraft")

from transformers import AutoImageProcessor

checkpoint = "microsoft/resnet-50"
image_processor = AutoImageProcessor.from_pretrained(checkpoint)

import re
from PIL import Image
import io

def contains_number(example):
    try:
        image = Image.open(io.BytesIO(example["image"]['bytes']))
        t = image_processor(images=image, return_tensors="pt")['pixel_values']
    except Exception as e:
        print(f"Error processing image：{example['text']}")
        return False
    return bool(re.search(r'\d', example['text']))

# Define a function to add the 'label' field
def add_label(example):
    lab = example['text'].split()
    temp = 'NOT'
    for item in lab:
        if str(item[-1]).isdigit():
            temp = item
            break
    example['label'] = temp
    return example

# Filter the dataset
# filtered_dataset = dataset.filter(contains_number)
# Add the 'label' field in the dataset
labeled_dataset = dataset.filter(contains_number).map(add_label)
# View the structure of the updated dataset
print(labeled_dataset)
```



### Expected behavior

needs to form labels
same as : https://www.kaggle.com/code/jiabaowangts/dataset-air/notebook

### Environment info

Kaggle notebook P100