ArrowNotImplementedError: Unsupported cast from string to list using function cast_list
Opened by divyakrishna-devisetty (closed)

Hi,

I am trying to load a local csv dataset(similar to explodinggradients_fiqa) using load_dataset. When I try to pass features, I am facing the mentioned issue.

CSV Data sample(golden_dataset.csv):
Question        |          Context                      |         answer      |        groundtruth
"what is abc?"   | "abc is this and that"     |    "abc is this "  |    "abc is this and that"

```
import csv 

# built it based on https://huggingface.co/datasets/explodinggradients/fiqa/viewer/ragas_eval?row=0
mydict = [
{'question' : "what is abc?", 'contexts': ["abc is this and that"], 'answer': "abc is this " , 'groundtruth':  ["abc is this and that"]},
{'question' : "what is abc?", 'contexts': ["abc is this and that"], 'answer': "abc is this " , 'groundtruth':  ["abc is this and that"]},
{'question' : "what is abc?", 'contexts': ["abc is this and that"], 'answer': "abc is this " , 'groundtruth':  ["abc is this and that"]}
]
         
fields = ['question', 'contexts', 'answer', 'ground_truths'] 

with open('golden_dataset.csv', 'w', newline='\n') as file: 
    writer = csv.DictWriter(file, fieldnames = fields)
    
    writer.writeheader() 
    for row in mydict:
        writer.writerow(row)
```

Retrieved dataset:
DatasetDict({
    train: Dataset({
        features: ['question', 'contexts', 'answer', 'ground_truths'],
        num_rows: 1
    })
})


Code to reproduce issue:


```
from datasets import load_dataset, Features, Sequence, Value

encode_features = Features(
    {
        "question": Value(dtype='string', id=0),
        "contexts": Sequence(feature=Value(dtype='string', id=1)),
        "answer": Value(dtype='string', id=2),
        "ground_truths": Sequence(feature=Value(dtype='string',id=3)),
    }
)

eval_dataset = load_dataset('csv', data_files='/golden_dataset.csv', features = encode_features )
```


Error trace:

```
---------------------------------------------------------------------------
ArrowNotImplementedError                  Traceback (most recent call last)
File ~/anaconda3/envs/python3/lib/python3.10/site-packages/datasets/builder.py:1925, in ArrowBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, job_id)
   1924 _time = time.time()
-> 1925 for _, table in generator:
   1926     if max_shard_size is not None and writer._num_bytes > max_shard_size:

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/datasets/packaged_modules/csv/csv.py:192, in Csv._generate_tables(self, files)
    189         # Uncomment for debugging (will print the Arrow table size and elements)
    190         # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
    191         # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
--> 192         yield (file_idx, batch_idx), self._cast_table(pa_table)
    193 except ValueError as e:

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/datasets/packaged_modules/csv/csv.py:167, in Csv._cast_table(self, pa_table)
    165 if all(not require_storage_cast(feature) for feature in self.config.features.values()):
    166     # cheaper cast
--> 167     pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
    168 else:
    169     # more expensive cast; allows str <-> int/float or str to Audio for example

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/table.pxi:3781, in pyarrow.lib.Table.from_arrays()

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/table.pxi:1449, in pyarrow.lib._sanitize_arrays()

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/array.pxi:354, in pyarrow.lib.asarray()

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/table.pxi:551, in pyarrow.lib.ChunkedArray.cast()

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/compute.py:400, in cast(arr, target_type, safe, options, memory_pool)
    399         options = CastOptions.safe(target_type)
--> 400 return call_function("cast", [arr], options, memory_pool)

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/_compute.pyx:572, in pyarrow._compute.call_function()

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/_compute.pyx:367, in pyarrow._compute.Function.call()

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/error.pxi:144, in pyarrow.lib.pyarrow_internal_check_status()

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/pyarrow/error.pxi:121, in pyarrow.lib.check_status()

ArrowNotImplementedError: Unsupported cast from string to list using function cast_list

The above exception was the direct cause of the following exception:

DatasetGenerationError                    Traceback (most recent call last)
Cell In[57], line 1
----> 1 eval_dataset = load_dataset('csv', data_files='/golden_dataset.csv', features = encode_features )
     

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/datasets/load.py:2153, in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, verification_mode, ignore_verifications, keep_in_memory, save_infos, revision, token, use_auth_token, task, streaming, num_proc, storage_options, **config_kwargs)
   2150 try_from_hf_gcs = path not in _PACKAGED_DATASETS_MODULES
   2152 # Download and prepare data
-> 2153 builder_instance.download_and_prepare(
   2154     download_config=download_config,
   2155     download_mode=download_mode,
   2156     verification_mode=verification_mode,
   2157     try_from_hf_gcs=try_from_hf_gcs,
   2158     num_proc=num_proc,
   2159     storage_options=storage_options,
   2160 )
   2162 # Build dataset for splits
   2163 keep_in_memory = (
   2164     keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size)
   2165 )

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/datasets/builder.py:954, in DatasetBuilder.download_and_prepare(self, output_dir, download_config, download_mode, verification_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, file_format, max_shard_size, num_proc, storage_options, **download_and_prepare_kwargs)
    952     if num_proc is not None:
    953         prepare_split_kwargs["num_proc"] = num_proc
--> 954     self._download_and_prepare(
    955         dl_manager=dl_manager,
    956         verification_mode=verification_mode,
    957         **prepare_split_kwargs,
    958         **download_and_prepare_kwargs,
    959     )
    960 # Sync info
    961 self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/datasets/builder.py:1049, in DatasetBuilder._download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs)
   1045 split_dict.add(split_generator.split_info)
   1047 try:
   1048     # Prepare split will record examples associated to the split
-> 1049     self._prepare_split(split_generator, **prepare_split_kwargs)
   1050 except OSError as e:
   1051     raise OSError(
   1052         "Cannot find data file. "
   1053         + (self.manual_download_instructions or "")
   1054         + "\nOriginal error:\n"
   1055         + str(e)
   1056     ) from None

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/datasets/builder.py:1813, in ArrowBasedBuilder._prepare_split(self, split_generator, file_format, num_proc, max_shard_size)
   1811 job_id = 0
   1812 with pbar:
-> 1813     for job_id, done, content in self._prepare_split_single(
   1814         gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
   1815     ):
   1816         if done:
   1817             result = content

File ~/anaconda3/envs/python3/lib/python3.10/site-packages/datasets/builder.py:1958, in ArrowBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, job_id)
   1956     if isinstance(e, SchemaInferenceError) and e.__context__ is not None:
   1957         e = e.__context__
-> 1958     raise DatasetGenerationError("An error occurred while generating the dataset") from e
   1960 yield job_id, True, (total_num_examples, total_num_bytes, writer._features, num_shards, shard_lengths)

DatasetGenerationError: An error occurred while generating the dataset 
```
Environment Info:
datasets version: 2.14.5
Python version: 3.10.8
PyArrow version: 12.0.1
Pandas version: 2.0.3

I have also tried to load dataset first and then use cast_column, or save_to_disk and load_from_disk.

Comments:
You can use the following code to load this CSV with the list values preserved:
```python
from datasets import load_dataset
import ast

converters = {
    "contexts" : ast.literal_eval,
    "ground_truths" : ast.literal_eval,
}

ds = load_dataset("csv", data_files="golden_dataset.csv", converters=converters)
```

Thank you! it worked :)