# Buggy code
def _init_samples_in_baskets(self):
    chunks_to_process = int(len(self.baskets) / self.multiprocessing_chunk_size)
    num_cpus = min(mp.cpu_count(), self.max_processes, chunks_to_process) or 1

    logger.info(
        f"Got ya {num_cpus} parallel workers to fill the baskets with samples (chunksize = {self.multiprocessing_chunk_size})..."
    )

    with mp.Pool(processes=num_cpus) as p:
        with mp.Manager() as manager:
            if self.share_all_baskets_for_multiprocessing:
                all_dicts = manager.list([b.raw for b in self.baskets])
            else:
                all_dicts = None

            with mp.Pool(processes=num_cpus) as p:
                samples = p.imap(
                    partial(self._multiproc_sample, all_dicts=all_dicts),
                    self.baskets,
                    chunksize=self.multiprocessing_chunk_size,
                )

                for s, b in tqdm(zip(samples, self.baskets), total=len(self.baskets)):
                    b.samples = s


# Traceback (AttributeError)
08/28/2019 07:47:35 - INFO - farm.utils -   device: cuda n_gpu: 1, distributed training: False, 16-bits training: False
08/28/2019 07:47:35 - INFO - pytorch_transformers.tokenization_utils -   loading file https://int-deepset-models-bert.s3.eu-central-1.amazonaws.com/pytorch/bert-base-german-cased-vocab.txt from cache at C:\Users\JulianGerhard\.cache\torch\pytorch_transformers\da299cdd121a3d71e1626f2908dda0d02658f42e925a3d6abd8273ec08cf41a6.2a48e6c60dcdb582effb718237ce5894652e3b4abb94f0a4d9a857b70333308d
08/28/2019 07:47:35 - INFO - farm.data_handler.data_silo -
Loading data into the data silo ...
______
|o  |   !
__          |:`_|---'-.
|__|______.-/ _ \-----.|
(o)(o)------'\ _ /     ( )

08/28/2019 07:47:35 - INFO - farm.data_handler.data_silo -   Loading train set from: data/conll03-de\train.txt
08/28/2019 07:47:35 - INFO - farm.data_handler.utils -    Couldn't find data/conll03-de\train.txt locally. Trying to download ...
08/28/2019 07:47:35 - INFO - farm.data_handler.utils -   downloading and extracting file C:\Users\JulianGerhard\PycharmProjects\word_embeddings\ner_finetuning\data\conll03-de to dir
08/28/2019 07:47:39 - INFO - farm.data_handler.processor -   Got ya 8 parallel workers to fill the baskets with samples (chunksize = 1000)...
0%|          | 0/24000 [00:00<?, ?it/s]multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\processor.py", line 310, in _multiproc_sample
samples = cls._dict_to_samples(dict=basket.raw, all_dicts=all_dicts)
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\processor.py", line 555, in _dict_to_samples
tokenized = tokenize_with_metadata(dict["text"], cls.tokenizer, cls.max_seq_len)
AttributeError: type object 'NERProcessor' has no attribute 'tokenizer'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "C:/Users/JulianGerhard/PycharmProjects/word_embeddings/ner_finetuning/farm_experimen t.py", line 6, in <module>
run_experiment(experiments[0])
File "c:\users\juliangerhard\pycharmprojects\farm\farm\experiment.py", line 87, in run_experiment
distributed=distributed,
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\data_silo.py", line 39, in __init__
self._load_data()
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\data_silo.py", line 47, in _load_data
self.data["train"], self.tensor_names = self.processor.dataset_from_file(train_file)
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\processor.py", line 366, in dataset_from_file
self._init_samples_in_baskets()
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\processor.py", line 304, in _init_samples_in_baskets
zip(samples, self.baskets), total=len(self.baskets)
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\site-packages\tqdm\_tqdm.py", line 1034, in __iter__
for obj in iterable:
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 320, in <genexpr>
return (item for chunk in result for item in chunk)
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 735, in next
raise value
AttributeError: type object 'NERProcessor' has no attribute 'tokenizer'
0%|          | 0/24000 [00:06<?, ?it/s]

# Fixed code
def _init_samples_in_baskets(self):
    with ExitStack() as stack:
        if self.use_multiprocessing:
            chunks_to_process = int(len(self.baskets) / self.multiprocessing_chunk_size)
            num_cpus = min(mp.cpu_count(), self.max_processes, chunks_to_process) or 1

            logger.info(
                f"Got ya {num_cpus} parallel workers to fill the baskets with samples (chunksize = {self.multiprocessing_chunk_size})..."
            )
            p = stack.enter_context(mp.Pool(processes=num_cpus))
            manager = stack.enter_context(mp.Manager())

            if self.share_all_baskets_for_multiprocessing:
                all_dicts = manager.list([b.raw for b in self.baskets])
            else:
                all_dicts = None

            samples = p.imap(
                partial(self._multiproc_sample, all_dicts=all_dicts),
                self.baskets,
                chunksize=self.multiprocessing_chunk_size,
            )
        else:
            all_dicts = [b.raw for b in self.baskets]
            samples = map(
                partial(self._multiproc_sample, all_dicts=all_dicts), self.baskets
            )

        for s, b in tqdm(zip(samples, self.baskets), total=len(self.baskets)):
            b.samples = s