Windows Permission Error (most recent version of datasets)
State: closed | Author: itsLuisa (NONE)

Hi everyone,
Can anyone help me with why the dataset loading script below raises a Windows Permission Error? I stuck quite closely to https://github.com/huggingface/datasets/blob/master/datasets/conll2003/conll2003.py , only I want to load the data from three local three-column tsv-files (id\ttokens\tpos_tags\n). I am using the most recent version of datasets. Thank you in advance!
Luisa

My script:
```
import datasets
import csv

logger = datasets.logging.get_logger(__name__)


class SampleConfig(datasets.BuilderConfig):

    def __init__(self, **kwargs):
        super(SampleConfig, self).__init__(**kwargs)


class Sample(datasets.GeneratorBasedBuilder):
    BUILDER_CONFIGS = [
        SampleConfig(name="conll2003", version=datasets.Version("1.0.0"), description="Conll2003 dataset"),
    ]

    def _info(self):
        return datasets.DatasetInfo(
            description="Dataset with words and their POS-Tags",
            features=datasets.Features(
                {
                    "id": datasets.Value("string"),
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "pos_tags": datasets.Sequence(
                        datasets.features.ClassLabel(
                            names=[
                                "''",
                                ",",
                                "-LRB-",
                                "-RRB-",
                                ".",
                                ":",
                                "CC",
                                "CD",
                                "DT",
                                "EX",
                                "FW",
                                "HYPH",
                                "IN",
                                "JJ",
                                "JJR",
                                "JJS",
                                "MD",
                                "NN",
                                "NNP",
                                "NNPS",
                                "NNS",
                                "PDT",
                                "POS",
                                "PRP",
                                "PRP$",
                                "RB",
                                "RBR",
                                "RBS",
                                "RP",
                                "TO",
                                "UH",
                                "VB",
                                "VBD",
                                "VBG",
                                "VBN",
                                "VBP",
                                "VBZ",
                                "WDT",
                                "WP",
                                "WRB",
                                "``"
                            ]
                        )
                    ),
                }
            ),
            supervised_keys=None,
            homepage="https://catalog.ldc.upenn.edu/LDC2011T03",
            citation="Weischedel, Ralph, et al. OntoNotes Release 4.0 LDC2011T03. Web Download. Philadelphia: Linguistic Data Consortium, 2011.",
        )

    def _split_generators(self, dl_manager):
        loaded_files = dl_manager.download_and_extract(self.config.data_files)
        return [
            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": loaded_files["train"]}),
            datasets.SplitGenerator(name=datasets.Split.TEST, gen_kwargs={"filepath": loaded_files["test"]}),
            datasets.SplitGenerator(name=datasets.Split.VALIDATION, gen_kwargs={"filepath": loaded_files["val"]})
        ]

    def _generate_examples(self, filepath):
        logger.info("generating examples from = %s", filepath)
        with open(filepath, encoding="cp1252") as f:
            data = csv.reader(f, delimiter="\t")
            ids = list()
            tokens = list()
            pos_tags = list()
            for id_, line in enumerate(data):
                #print(line)
                if len(line) == 1:
                    if tokens:
                        yield id_, {"id": ids, "tokens": tokens, "pos_tags": pos_tags}
                        ids = list()
                        tokens = list()
                        pos_tags = list()
                else:
                    ids.append(line[0])
                    tokens.append(line[1])
                    pos_tags.append(line[2])
            # last example
            yield id_, {"id": ids, "tokens": tokens, "pos_tags": pos_tags}


def main():
    dataset = datasets.load_dataset(
        "data_loading.py", data_files={
            "train": "train.tsv",
            "test": "test.tsv",
            "val": "val.tsv"
        }
    )

    #print(dataset)

if __name__=="__main__":
    main()
```


Comments: Hi @itsLuisa !

Could you give us more information about the error you're getting, please?
A copy-paste of the Traceback would be nice to get a better understanding of what is wrong :) 

Hello @SBrandeis , this is it:
```
Traceback (most recent call last):
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 537, in incomplete_dir
    yield tmp_dir
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 578, in download_and_prepare
    self._download_and_prepare(
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 656, in _download_and_prepare
    self._prepare_split(split_generator, **prepare_split_kwargs)
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 982, in _prepare_split
    num_examples, num_bytes = writer.finalize()
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\arrow_writer.py", line 297, in finalize
    self.write_on_file()
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\arrow_writer.py", line 230, in write_on_file
    pa_array = pa.array(typed_sequence)
  File "pyarrow\array.pxi", line 222, in pyarrow.lib.array
  File "pyarrow\array.pxi", line 110, in pyarrow.lib._handle_arrow_array_protocol
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\arrow_writer.py", line 97, in __arrow_array__
    out = pa.array(self.data, type=type)
  File "pyarrow\array.pxi", line 305, in pyarrow.lib.array
  File "pyarrow\array.pxi", line 39, in pyarrow.lib._sequence_to_array
  File "pyarrow\error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow\error.pxi", line 107, in pyarrow.lib.check_status
pyarrow.lib.ArrowTypeError: Expected bytes, got a 'list' object

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:/Users/Luisa/Documents/Uni/WS 2020,21/Neural Networks/Final_Project/NN_Project/data_loading.py", line 122, in <module>
    main()
  File "C:/Users/Luisa/Documents/Uni/WS 2020,21/Neural Networks/Final_Project/NN_Project/data_loading.py", line 111, in main
    dataset = datasets.load_dataset(
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\load.py", line 740, in load_dataset
    builder_instance.download_and_prepare(
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 586, in download_and_prepare
    self._save_info()
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\contextlib.py", line 131, in __exit__
    self.gen.throw(type, value, traceback)
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\site-packages\datasets\builder.py", line 543, in incomplete_dir
    shutil.rmtree(tmp_dir)
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\shutil.py", line 740, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\shutil.py", line 618, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\Users\Luisa\AppData\Local\Programs\Python\Python38\lib\shutil.py", line 616, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 32] Der Prozess kann nicht auf die Datei zugreifen, da sie von einem anderen Prozess verwendet wird: 'C:\\Users\\Luisa\\.cache\\huggingface\\datasets\\sample\\default-20ee7d51a6a9454f\\0.0.0\\5fc4c3a355ea77ab446bd31fca5082437600b8364d29b2b95264048bd1f398b1.incomplete\\sample-train.arrow'

Process finished with exit code 1
```

Hi @itsLuisa, thanks for sharing the Traceback.

You are defining the "id" field as a `string` feature:
```python
class Sample(datasets.GeneratorBasedBuilder):
    ...

    def _info(self):
        return datasets.DatasetInfo(
            features=datasets.Features(
                {
                    "id": datasets.Value("string"),
                    # ^^ here
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "pos_tags": datasets.Sequence(datasets.features.ClassLabel(names=[...])),
[...]
```

But in the `_generate_examples`, the "id" field is a list:
```python
ids = list()
```

Changing:
```python
"id": datasets.Value("string"),
```
Into:
```python
"id": datasets.Sequence(datasets.Value("string")),
```

Should fix your issue.

Let me know if this helps!

It seems to be working now, thanks a lot for the help, @SBrandeis !

Glad to hear it!
I'm closing the issue