Custom Json Dataset Throwing Error when batch is False
Opened by jmandivarapu1 (closed)

## Describe the bug
A clear and concise description of what the bug is.
I tried to create my custom dataset using below code

```
from datasets import Features, Sequence, ClassLabel, Value, Array2D, Array3D
from torchvision import transforms
from transformers import AutoProcessor
# we'll use the Auto API here - it will load LayoutLMv3Processor behind the scenes,
# based on the checkpoint we provide from the hub
from datasets import load_dataset

def prepare_examples(examples):
     #Some preporcessing for each image and text as all my data saved in cloud
     #For this reason I couldn't set the batch to True. 
      encoding = processor(img_as_tensor, words, boxes=boxes, word_labels=labels,
                       truncation=True, padding="max_length")
    # encoding['pixel_values']=np.array(encoding['pixel_values'])
    return encoding

dataset = load_dataset("json", data_files='issues.jsonl')
processor = AutoProcessor.from_pretrained("microsoft/layoutlmv3-base", apply_ocr=False)
features = dataset["train"].features
column_names = dataset["train"].column_names
# we need to define custom features for `set_format` (used later on) to work properly
features = Features({
    'pixel_values': Array3D(dtype="float32", shape=(3, 224, 224)),
    'input_ids': Sequence(feature=Value(dtype='int64')),
    'attention_mask': Sequence(Value(dtype='int64')),
    'bbox': Array2D(dtype="int64", shape=(512, 4)),
    'labels': Sequence(feature=Value(dtype='int64')),
})

train_dataset = dataset["train"].map(
    prepare_examples,
    batched=False,
    remove_columns=column_names,
    features=features
)
```

It throws below error.
```
/opt/conda/lib/python3.7/site-packages/datasets/arrow_writer.py in __arrow_array__(self, type)
    172                 storage = to_pyarrow_listarray(data, pa_type)
--> 173                 return pa.ExtensionArray.from_storage(pa_type, storage)
    174 

/opt/conda/lib/python3.7/site-packages/pyarrow/array.pxi in pyarrow.lib.ExtensionArray.from_storage()

TypeError: Incompatible storage type list<item: list<item: list<item: list<item: float>>>> for extension type extension<arrow.py_extension_type<Array3DExtensionType>>
```
## Steps to reproduce the bug
```python
# Sample code to reproduce the bug
```
rom datasets import Features, Sequence, ClassLabel, Value, Array2D, Array3D
from torchvision import transforms
from transformers import AutoProcessor
# we'll use the Auto API here - it will load LayoutLMv3Processor behind the scenes,
# based on the checkpoint we provide from the hub
from datasets import load_dataset

def prepare_examples(examples):
     #Some preporcessing for each image and text as all my data saved in cloud
      encoding = processor(img_as_tensor, words, boxes=boxes, word_labels=labels,
                       truncation=True, padding="max_length")
    # encoding['pixel_values']=np.array(encoding['pixel_values'])
    return encoding

dataset = load_dataset("json", data_files='issues.jsonl')
processor = AutoProcessor.from_pretrained("microsoft/layoutlmv3-base", apply_ocr=False)
features = dataset["train"].features
column_names = dataset["train"].column_names
# we need to define custom features for `set_format` (used later on) to work properly
features = Features({
    'pixel_values': Array3D(dtype="float32", shape=(3, 224, 224)),
    'input_ids': Sequence(feature=Value(dtype='int64')),
    'attention_mask': Sequence(Value(dtype='int64')),
    'bbox': Array2D(dtype="int64", shape=(512, 4)),
    'labels': Sequence(feature=Value(dtype='int64')),
})

train_dataset = dataset["train"].map(
    prepare_examples,
    batched=False,
    remove_columns=column_names,
    features=features
)
## Expected results
A clear and concise description of the expected results.
Expected would be similar to all the otherdatasets with no error.
## Actual results
Specify the actual results or traceback.

## Environment info
<!-- You can run the command `datasets-cli env` and copy-and-paste its output below. -->
- `datasets` version:
- Platform: Unix
- Python version: 3.9
- PyArrow version: 9.0.0


Comments:
Hi! Our processors are meant to be used in `batched` mode, so if `batched` is `False`, you need to drop the batch dimension (the error message warns you that the array has an extra dimension meaning it's 4D instead of 3D) to avoid the error:
```python
def prepare_examples(examples):
    #Some preporcessing for each image and text as all my data saved in cloud
    #For this reason I couldn't set the batch to True. 
    encoding = processor(img_as_tensor, words, boxes=boxes, word_labels=labels,
                       truncation=True, padding="max_length", return_tensors="np")
    # drop extra dim
    for k in encoding.items():
        encoding[k]=encoding[k][0]
    return encoding
```

> Hi! Our processors are meant to be used in `batched` mode, so if `batched` is `False`, you need to drop the batch dimension (the error message warns you that the array has an extra dimension meaning it's 4D instead of 3D) to avoid the error:
> 
> ```python
> def prepare_examples(examples):
>     #Some preporcessing for each image and text as all my data saved in cloud
>     #For this reason I couldn't set the batch to True. 
>     encoding = processor(img_as_tensor, words, boxes=boxes, word_labels=labels,
>                        truncation=True, padding="max_length", return_tensors="np")
>     # drop extra dim
>     for k in encoding.items():
>         encoding[k]=encoding[k][0]
>     return encoding
> ```

Thank you it did work

```
for k,v in encoding.items():
        encoding[k]=encoding[k][0]
```