# Buggy code
def __init__(
    self,
    tokenizer,
    max_seq_len,
    label_list,
    metrics,
    train_filename,
    dev_filename,
    test_filename,
    dev_split,
    data_dir,
    label_dtype=torch.long,
    multiprocessing_chunk_size=1_000,
    max_processes=128,
    share_all_baskets_for_multiprocessing=False,
):
    """
    Initialize a generic Processor

    :param tokenizer: Used to split a sentence (str) into tokens.
    :param max_seq_len: Samples are truncated after this many tokens.
    :type max_seq_len: int
    :param label_list: List of all unique target labels.
    :type label_list: list
    :param metrics: The metric used for evaluation, one per prediction head.
                    Choose from mcc, acc, acc_f1, pear_spear, seq_f1, f1_macro, squad.
    :type metrics: list or str
    :param train_filename: The name of the file containing training data.
    :type train_filename: str
    :param dev_filename: The name of the file containing the dev data. If None and 0.0 < dev_split < 1.0 the dev set
                         will be a slice of the train set.
    :type dev_filename: str or None
    :param test_filename: The name of the file containing test data.
    :type test_filename: str
    :param dev_split: The proportion of the train set that will sliced. Only works if dev_filename is set to None
    :type dev_split: float
    :param data_dir: The directory in which the train, test and perhaps dev files can be found.
    :type data_dir: str
    :param label_dtype: The torch dtype for the labels.
    :param max_processes: maximum number of processing to use for Multiprocessing.
    :type max_processes: int
    """

    # The Multiprocessing functions in the Class are classmethods to avoid passing(and pickling) of class-objects
    # that are very large in size(eg, self.baskets). Since classmethods have access to only class attributes, all
    # objects required in Multiprocessing must be set as class attributes.
    Processor.tokenizer = tokenizer
    Processor.max_seq_len = max_seq_len
    Processor.label_list = label_list

    # data sets
    self.train_filename = train_filename
    self.dev_filename = dev_filename
    self.test_filename = test_filename
    self.dev_split = dev_split
    self.data_dir = data_dir
    # labels
    self.label_dtype = label_dtype
    self.label_maps = []
    # multiprocessing
    self.multiprocessing_chunk_size = multiprocessing_chunk_size
    self.share_all_baskets_for_multiprocessing = share_all_baskets_for_multiprocessing
    self.max_processes = max_processes
    # others
    self.metrics = [metrics] if isinstance(metrics, str) else metrics

    # create label maps (one per prediction head)
    if any(isinstance(i, list) for i in label_list):
        for labels_per_head in label_list:
            map = {i: label for i, label in enumerate(labels_per_head)}
            self.label_maps.append(map)
    else:
        map = {i: label for i, label in enumerate(label_list)}
        self.label_maps.append(map)

    self.baskets = []

    self._log_params()


# Traceback (AttributeError)
08/28/2019 07:47:35 - INFO - farm.utils -   device: cuda n_gpu: 1, distributed training: False, 16-bits training: False
08/28/2019 07:47:35 - INFO - pytorch_transformers.tokenization_utils -   loading file https://int-deepset-models-bert.s3.eu-central-1.amazonaws.com/pytorch/bert-base-german-cased-vocab.txt from cache at C:\Users\JulianGerhard\.cache\torch\pytorch_transformers\da299cdd121a3d71e1626f2908dda0d02658f42e925a3d6abd8273ec08cf41a6.2a48e6c60dcdb582effb718237ce5894652e3b4abb94f0a4d9a857b70333308d
08/28/2019 07:47:35 - INFO - farm.data_handler.data_silo -
Loading data into the data silo ...
______
|o  |   !
__          |:`_|---'-.
|__|______.-/ _ \-----.|
(o)(o)------'\ _ /     ( )

08/28/2019 07:47:35 - INFO - farm.data_handler.data_silo -   Loading train set from: data/conll03-de\train.txt
08/28/2019 07:47:35 - INFO - farm.data_handler.utils -    Couldn't find data/conll03-de\train.txt locally. Trying to download ...
08/28/2019 07:47:35 - INFO - farm.data_handler.utils -   downloading and extracting file C:\Users\JulianGerhard\PycharmProjects\word_embeddings\ner_finetuning\data\conll03-de to dir
08/28/2019 07:47:39 - INFO - farm.data_handler.processor -   Got ya 8 parallel workers to fill the baskets with samples (chunksize = 1000)...
0%|          | 0/24000 [00:00<?, ?it/s]multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\processor.py", line 310, in _multiproc_sample
samples = cls._dict_to_samples(dict=basket.raw, all_dicts=all_dicts)
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\processor.py", line 555, in _dict_to_samples
tokenized = tokenize_with_metadata(dict["text"], cls.tokenizer, cls.max_seq_len)
AttributeError: type object 'NERProcessor' has no attribute 'tokenizer'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "C:/Users/JulianGerhard/PycharmProjects/word_embeddings/ner_finetuning/farm_experimen t.py", line 6, in <module>
run_experiment(experiments[0])
File "c:\users\juliangerhard\pycharmprojects\farm\farm\experiment.py", line 87, in run_experiment
distributed=distributed,
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\data_silo.py", line 39, in __init__
self._load_data()
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\data_silo.py", line 47, in _load_data
self.data["train"], self.tensor_names = self.processor.dataset_from_file(train_file)
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\processor.py", line 366, in dataset_from_file
self._init_samples_in_baskets()
File "c:\users\juliangerhard\pycharmprojects\farm\farm\data_handler\processor.py", line 304, in _init_samples_in_baskets
zip(samples, self.baskets), total=len(self.baskets)
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\site-packages\tqdm\_tqdm.py", line 1034, in __iter__
for obj in iterable:
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 320, in <genexpr>
return (item for chunk in result for item in chunk)
File "C:\Users\JulianGerhard\AppData\Local\Programs\Python\Python36\lib\multiprocessing\pool.py", line 735, in next
raise value
AttributeError: type object 'NERProcessor' has no attribute 'tokenizer'
0%|          | 0/24000 [00:06<?, ?it/s]

# Fixed code
def __init__(
    self,
    tokenizer,
    max_seq_len,
    label_list,
    metrics,
    train_filename,
    dev_filename,
    test_filename,
    dev_split,
    data_dir,
    label_dtype=torch.long,
    multiprocessing_chunk_size=1_000,
    max_processes=128,
    share_all_baskets_for_multiprocessing=False,
    use_multiprocessing=True,
):
    """
    Initialize a generic Processor

    :param tokenizer: Used to split a sentence (str) into tokens.
    :param max_seq_len: Samples are truncated after this many tokens.
    :type max_seq_len: int
    :param label_list: List of all unique target labels.
    :type label_list: list
    :param metrics: The metric used for evaluation, one per prediction head.
                    Choose from mcc, acc, acc_f1, pear_spear, seq_f1, f1_macro, squad.
    :type metrics: list or str
    :param train_filename: The name of the file containing training data.
    :type train_filename: str
    :param dev_filename: The name of the file containing the dev data. If None and 0.0 < dev_split < 1.0 the dev set
                         will be a slice of the train set.
    :type dev_filename: str or None
    :param test_filename: The name of the file containing test data.
    :type test_filename: str
    :param dev_split: The proportion of the train set that will sliced. Only works if dev_filename is set to None
    :type dev_split: float
    :param data_dir: The directory in which the train, test and perhaps dev files can be found.
    :type data_dir: str
    :param label_dtype: The torch dtype for the labels.
    :param max_processes: maximum number of processing to use for Multiprocessing.
    :type max_processes: int
    """

    # The Multiprocessing functions in the Class are classmethods to avoid passing(and pickling) of class-objects
    # that are very large in size(eg, self.baskets). Since classmethods have access to only class attributes, all
    # objects required in Multiprocessing must be set as class attributes.
    Processor.tokenizer = tokenizer
    Processor.max_seq_len = max_seq_len
    Processor.label_list = label_list

    # data sets
    self.train_filename = train_filename
    self.dev_filename = dev_filename
    self.test_filename = test_filename
    self.dev_split = dev_split
    self.data_dir = data_dir
    # labels
    self.label_dtype = label_dtype
    self.label_maps = []
    # multiprocessing
    if os.name == "nt":
        self.use_multiprocessing = (
            False  # the mp code here isn't compatible with Windows
        )
    else:
        self.use_multiprocessing = use_multiprocessing
    self.multiprocessing_chunk_size = multiprocessing_chunk_size
    self.share_all_baskets_for_multiprocessing = share_all_baskets_for_multiprocessing
    self.max_processes = max_processes
    # others
    self.metrics = [metrics] if isinstance(metrics, str) else metrics

    # create label maps (one per prediction head)
    if any(isinstance(i, list) for i in label_list):
        for labels_per_head in label_list:
            map = {i: label for i, label in enumerate(labels_per_head)}
            self.label_maps.append(map)
    else:
        map = {i: label for i, label in enumerate(label_list)}
        self.label_maps.append(map)

    self.baskets = []

    self._log_params()