Repository: deepset-ai/haystack

Issue:
Hardcoded maximum number of answers causes "index out of bounds error"
**Describe the bug**
It seems to be the case, when the question have more than 6 answers it can (and do) cause the "index out of bounds error" in some scenarios

**Error message**
---------------------------------------------------------------------------

IndexError                                Traceback (most recent call last)

[<ipython-input-4-43de0b14cfe8>](https://localhost:8080/#) in <module>
----> 1 reader.train(data_dir=data_dir, 
      2             train_filename="train.json",
      3             test_filename="test.json",
      4             dev_filename="dev.json",
      5             use_gpu=True,

6 frames

[/usr/local/lib/python3.8/dist-packages/haystack/nodes/reader/farm.py](https://localhost:8080/#) in train(self, data_dir, train_filename, dev_filename, test_filename, use_gpu, devices, batch_size, n_epochs, learning_rate, max_seq_len, warmup_proportion, dev_split, evaluate_every, save_dir, num_processes, use_amp, checkpoint_root_dir, checkpoint_every, checkpoints_to_keep, caching, cache_path, grad_acc_steps, early_stopping, max_query_length)
    435         :return: None
    436         """
--> 437         return self._training_procedure(
    438             data_dir=data_dir,
    439             train_filename=train_filename,

[/usr/local/lib/python3.8/dist-packages/haystack/nodes/reader/farm.py](https://localhost:8080/#) in _training_procedure(self, data_dir, train_filename, dev_filename, test_filename, use_gpu, devices, batch_size, n_epochs, learning_rate, max_seq_len, warmup_proportion, dev_split, evaluate_every, save_dir, num_processes, use_amp, checkpoint_root_dir, checkpoint_every, checkpoints_to_keep, teacher_model, teacher_batch_size, caching, cache_path, distillation_loss_weight, distillation_loss, temperature, tinybert, processor, grad_acc_steps, early_stopping, distributed, doc_stride, max_query_length)
    269             )
    270         else:  # caching would need too much memory for tinybert distillation so in that case we use the default data silo
--> 271             data_silo = DataSilo(
    272                 processor=processor,
    273                 batch_size=batch_size,

[/usr/local/lib/python3.8/dist-packages/haystack/modeling/data_handler/data_silo.py](https://localhost:8080/#) in __init__(self, processor, batch_size, eval_batch_size, distributed, automatic_loading, max_multiprocessing_chunksize, max_processes, multiprocessing_strategy, caching, cache_path)
    102             # In most cases we want to load all data automatically, but in some cases we rather want to do this
    103             # later or load from dicts instead of file
--> 104             self._load_data()
    105 
    106     def _get_dataset(self, filename: Optional[Union[str, Path]], dicts: Optional[List[Dict]] = None):

[/usr/local/lib/python3.8/dist-packages/haystack/modeling/data_handler/data_silo.py](https://localhost:8080/#) in _load_data(self, train_dicts, dev_dicts, test_dicts)
    163             train_file = self.processor.data_dir / self.processor.train_filename
    164             logger.info("Loading train set from: %s ", train_file)
--> 165             self.data["train"], self.tensor_names = self._get_dataset(train_file)
    166         else:
    167             logger.info("No train set is being loaded")

[/usr/local/lib/python3.8/dist-packages/haystack/modeling/data_handler/data_silo.py](https://localhost:8080/#) in _get_dataset(self, filename, dicts)
    123         for i in tqdm(range(0, num_dicts, batch_size), desc="Preprocessing dataset", unit=" Dicts"):
    124             processing_batch = dicts[i : i + batch_size]
--> 125             dataset, tensor_names, problematic_sample_ids = self.processor.dataset_from_dicts(
    126                 dicts=processing_batch, indices=list(range(len(processing_batch)))  # TODO remove indices
    127             )

[/usr/local/lib/python3.8/dist-packages/haystack/modeling/data_handler/processor.py](https://localhost:8080/#) in dataset_from_dicts(self, dicts, indices, return_baskets, debug)
    472         # Convert answers from string to token space, skip this step for inference
    473         if not return_baskets:
--> 474             baskets = self._convert_answers(baskets)
    475 
    476         # Convert internal representation (nested baskets + samples with mixed types) to pytorch features (arrays of numbers)

[/usr/local/lib/python3.8/dist-packages/haystack/modeling/data_handler/processor.py](https://localhost:8080/#) in _convert_answers(self, baskets)
    653                             print(self.max_answers)
    654                             print(i)
--> 655                             label_idxs[i][0] = self.sp_toks_start + question_len_t + self.sp_toks_mid + answer_start_t
    656                             label_idxs[i][1] = self.sp_toks_start + question_len_t + self.sp_toks_mid + answer_end_t
    657                         # If the start or end of the span answer is outside the passage, treat passage as no_answer

IndexError: index 6 is out of bounds for axis 0 with size 6

**Expected behavior**
SquadProcessor should be able to work with arbitrary number of answers to any question.

**Additional context**
I think the problem caused by hardcoding magic number in the init of `class SquadProcessor(Processor)`, where is set that `max_answers: int = 6,`

Perhaps, using `len(basket.raw["answers"])` each time could fix that problem

**To Reproduce**
Download PolicyQA dataset from https://github.com/wasiahmad/PolicyQA and launch 
```
reader = FARMReader(model_name_or_path="ahotrod/electra_large_discriminator_squad2_512", use_gpu=True)
data_dir = "/content/PolicyQA/data"
reader.train(data_dir=data_dir, 
            train_filename="train.json", 
            test_filename="test.json",
            dev_filename="dev.json",
            use_gpu=True, 
            n_epochs=1,
            batch_size = 1,
            max_seq_len = 512,             
            save_dir="my_model",
            )

```


**FAQ Check**
- [+] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?

**System:**
 - OS: linux
 - GPU/CPU: tesla v100
 - Haystack version (commit or version number): 1.14.0
 - DocumentStore: none
 - Reader: FARMReader(model_name_or_path="ahotrod/electra_large_discriminator_squad2_512", use_gpu=True)
 - Retriever: none



Discussion:
As a quickfix I chenged max_answers: int = 6 to max_answers: int = 10 and it works flawlessly now
Hey @Serbernari, thank you for reporting this! I can reproduce it as well. 
By the way, we accept contributions. Would you like to open a PR for your suggested solution? That way, it will be faster for us to fix it. If not, our team will take over this for sure  

Resolving patch:
diff --git a/haystack/modeling/data_handler/processor.py b/haystack/modeling/data_handler/processor.py
--- a/haystack/modeling/data_handler/processor.py
+++ b/haystack/modeling/data_handler/processor.py
@@ -35,7 +35,6 @@
 from haystack.modeling.data_handler.input_features import sample_to_features_text
 from haystack.utils.experiment_tracking import Tracker as tracker
 
-
 DOWNSTREAM_TASK_MAP = {
     "squad20": "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-downstream/squad20.tar.gz",
     "covidqa": "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-downstream/covidqa.tar.gz",
@@ -381,7 +380,7 @@ def __init__(
         doc_stride: int = 128,
         max_query_length: int = 64,
         proxies: Optional[dict] = None,
-        max_answers: int = 6,
+        max_answers: Optional[int] = None,
         **kwargs,
     ):
         """
@@ -403,7 +402,9 @@ def __init__(
         :param max_query_length: Maximum length of the question (in number of subword tokens)
         :param proxies: proxy configuration to allow downloads of remote datasets.
                         Format as in  "requests" library: https://2.python-requests.org//en/latest/user/advanced/#proxies
-        :param max_answers: number of answers to be converted. QA dev or train sets can contain multi-way annotations, which are converted to arrays of max_answer length
+        :param max_answers: Number of answers to be converted. QA sets can contain multi-way annotations, which are converted to arrays of max_answer length.
+                            Adjusts to maximum number of answers in the first processed datasets if not set.
+                            Truncates or pads to max_answer length if set.
         :param kwargs: placeholder for passing generic parameters
         """
         self.ph_output_type = "per_token_squad"
@@ -469,12 +470,19 @@ def dataset_from_dicts(
         # Split documents into smaller passages to fit max_seq_len
         baskets = self._split_docs_into_passages(baskets)
 
+        # Determine max_answers if not set
+        max_answers = (
+            self.max_answers
+            if self.max_answers is not None
+            else max(max(len(basket.raw["answers"]) for basket in baskets), 1)
+        )
+
         # Convert answers from string to token space, skip this step for inference
         if not return_baskets:
-            baskets = self._convert_answers(baskets)
+            baskets = self._convert_answers(baskets, max_answers)
 
         # Convert internal representation (nested baskets + samples with mixed types) to pytorch features (arrays of numbers)
-        baskets = self._passages_to_pytorch_features(baskets, return_baskets)
+        baskets = self._passages_to_pytorch_features(baskets, return_baskets, max_answers)
 
         # Convert features into pytorch dataset, this step also removes potential errors during preprocessing
         dataset, tensor_names, baskets = self._create_dataset(baskets)
@@ -607,7 +615,7 @@ def _split_docs_into_passages(self, baskets: List[SampleBasket]):
 
         return baskets
 
-    def _convert_answers(self, baskets: List[SampleBasket]):
+    def _convert_answers(self, baskets: List[SampleBasket], max_answers: int):
         """
         Converts answers that are pure strings into the token based representation with start and end token offset.
         Can handle multiple answers per question document pair as is common for development/text sets
@@ -617,7 +625,7 @@ def _convert_answers(self, baskets: List[SampleBasket]):
             for sample in basket.samples:  # type: ignore
                 # Dealing with potentially multiple answers (e.g. Squad dev set)
                 # Initializing a numpy array of shape (max_answers, 2), filled with -1 for missing values
-                label_idxs = np.full((self.max_answers, 2), fill_value=-1)
+                label_idxs = np.full((max_answers, 2), fill_value=-1)
 
                 if error_in_answer or (len(basket.raw["answers"]) == 0):
                     # If there are no answers we set
@@ -625,6 +633,14 @@ def _convert_answers(self, baskets: List[SampleBasket]):
                 else:
                     # For all other cases we use start and end token indices, that are relative to the passage
                     for i, answer in enumerate(basket.raw["answers"]):
+                        if i >= max_answers:
+                            logger.warning(
+                                "Found a sample with more answers (%d) than "
+                                "max_answers (%d). These will be ignored.",
+                                len(basket.raw["answers"]),
+                                max_answers,
+                            )
+                            break
                         # Calculate start and end relative to document
                         answer_len_c = len(answer["text"])
                         answer_start_c = answer["answer_start"]
@@ -691,7 +707,7 @@ def _convert_answers(self, baskets: List[SampleBasket]):
 
         return baskets
 
-    def _passages_to_pytorch_features(self, baskets: List[SampleBasket], return_baskets: bool):
+    def _passages_to_pytorch_features(self, baskets: List[SampleBasket], return_baskets: bool, max_answers: int):
         """
         Convert internal representation (nested baskets + samples with mixed types) to python features (arrays of numbers).
         We first join question and passages into one large vector.
@@ -769,7 +785,7 @@ def _passages_to_pytorch_features(self, baskets: List[SampleBasket], return_bask
                     len(input_ids) == len(padding_mask) == len(segment_ids) == len(start_of_word) == len(span_mask)
                 )
                 id_check = len(sample_id) == 3
-                label_check = return_baskets or len(sample.tokenized.get("labels", [])) == self.max_answers  # type: ignore
+                label_check = return_baskets or len(sample.tokenized.get("labels", [])) == max_answers  # type: ignore
                 # labels are set to -100 when answer cannot be found
                 label_check2 = return_baskets or np.all(sample.tokenized["labels"] > -99)  # type: ignore
                 if len_check and id_check and label_check and label_check2:


Test patch:
diff --git a/test/modeling/test_processor.py b/test/modeling/test_processor.py
--- a/test/modeling/test_processor.py
+++ b/test/modeling/test_processor.py
@@ -1,5 +1,7 @@
+import copy
 import logging
 
+import pytest
 from transformers import AutoTokenizer
 
 from haystack.modeling.data_handler.processor import SquadProcessor
@@ -233,7 +235,7 @@ def test_batch_encoding_flatten_rename():
         pass
 
 
-def test_dataset_from_dicts_qa_labelconversion(samples_path, caplog=None):
+def test_dataset_from_dicts_qa_label_conversion(samples_path, caplog=None):
     if caplog:
         caplog.set_level(logging.CRITICAL)
 
@@ -248,7 +250,7 @@ def test_dataset_from_dicts_qa_labelconversion(samples_path, caplog=None):
 
     for model in models:
         tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model)
-        processor = SquadProcessor(tokenizer, max_seq_len=256, data_dir=None)
+        processor = SquadProcessor(tokenizer, max_seq_len=256, data_dir=None, max_answers=6)
 
         for sample_type in sample_types:
             dicts = processor.file_to_dicts(samples_path / "qa" / f"{sample_type}.json")
@@ -296,3 +298,36 @@ def test_dataset_from_dicts_qa_labelconversion(samples_path, caplog=None):
                         12,
                         12,
                     ], f"Processing labels for {model} has changed."
+
+
+@pytest.mark.integration
+def test_dataset_from_dicts_auto_determine_max_answers(samples_path, caplog=None):
+    """
+    SquadProcessor should determine the number of answers for the pytorch dataset based on
+    the maximum number of answers for each question. Vanilla.json has one question with two answers,
+    so the number of answers should be two.
+    """
+    model = "deepset/roberta-base-squad2"
+    tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model)
+    processor = SquadProcessor(tokenizer, max_seq_len=256, data_dir=None)
+    dicts = processor.file_to_dicts(samples_path / "qa" / "vanilla.json")
+    dataset, tensor_names, problematic_sample_ids = processor.dataset_from_dicts(dicts, indices=[1])
+    assert len(dataset[0][tensor_names.index("labels")]) == 2
+    # check that a max_answers will be adjusted when processing a different dataset with the same SquadProcessor
+    dicts_more_answers = copy.deepcopy(dicts)
+    dicts_more_answers[0]["qas"][0]["answers"] = dicts_more_answers[0]["qas"][0]["answers"] * 3
+    dataset, tensor_names, problematic_sample_ids = processor.dataset_from_dicts(dicts_more_answers, indices=[1])
+    assert len(dataset[0][tensor_names.index("labels")]) == 6
+
+
+@pytest.mark.integration
+def test_dataset_from_dicts_truncate_max_answers(samples_path, caplog=None):
+    """
+    Test that it is possible to manually set the number of answers, truncating the answers in the data.
+    """
+    model = "deepset/roberta-base-squad2"
+    tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=model)
+    processor = SquadProcessor(tokenizer, max_seq_len=256, data_dir=None, max_answers=1)
+    dicts = processor.file_to_dicts(samples_path / "qa" / "vanilla.json")
+    dataset, tensor_names, problematic_sample_ids = processor.dataset_from_dicts(dicts, indices=[1])
+    assert len(dataset[0][tensor_names.index("labels")]) == 1