Repository: deepset-ai/haystack

Issue:
Remove linearized offsets for table answers
The linearized table offsets are deprecated with version 1.16 and need to be removed with 1.18. Deprecation PR: https://github.com/deepset-ai/haystack/pull/4616

Extra Notes:

- Related issue about tutorials changes to be made: https://github.com/deepset-ai/haystack-tutorials/issues/169
- Deprecation notice can be found [here](https://docs.haystack.deepset.ai/docs/table_qa#tablereader) on our docs


Discussion:

Resolving patch:
diff --git a/haystack/nodes/reader/table.py b/haystack/nodes/reader/table.py
--- a/haystack/nodes/reader/table.py
+++ b/haystack/nodes/reader/table.py
@@ -73,7 +73,6 @@ def __init__(
         max_seq_len: int = 256,
         use_auth_token: Optional[Union[str, bool]] = None,
         devices: Optional[List[Union[str, "torch.device"]]] = None,
-        return_table_cell: bool = False,
     ):
         """
         Load a TableQA model from Transformers.
@@ -115,24 +114,10 @@ def __init__(
                         A list containing torch device objects and/or strings is supported (For example
                         [torch.device('cuda:0'), "mps", "cuda:1"]). When specifying `use_gpu=False` the devices
                         parameter is not used and a single cpu device is used for inference.
-        :param return_table_cell: Whether to return the offsets (`offsets_in_document` and `offsets_in_context`) that indicate
-                                  the table cells that answer the question using the TableCell schema. The TableCell schema returns the row
-                                  and column indices of the table cells selected in the Answer. Otherwise, the offsets
-                                  are returned as Span objects which are start and end indices when counting through the
-                                  table in a linear fashion, which means the first cell is top left and the last cell is bottom right.
         """
         torch_and_transformers_import.check()
         super().__init__()
 
-        if not return_table_cell:
-            logger.warning(
-                "The support for returning offsets in answer predictions in a linear fashion is being deprecated."
-                " Set return_table_cell=True to use the new offsets format which returns the row and column indices"
-                " of the table cells selected in the answer."
-                " In the future, return_table_cell=True will become default and return_table_cell=False will no "
-                " longer be supported."
-            )
-
         self.devices, _ = initialize_device_settings(devices=devices, use_cuda=use_gpu, multi_gpu=False)
         if len(self.devices) > 1:
             logger.warning(
@@ -151,7 +136,6 @@ def __init__(
                 tokenizer=tokenizer,
                 max_seq_len=max_seq_len,
                 use_auth_token=use_auth_token,
-                return_table_cell=return_table_cell,
             )
         elif config.architectures[0] == "TapasForScoredQA":
             self.table_encoder = _TapasScoredEncoder(
@@ -163,7 +147,6 @@ def __init__(
                 return_no_answer=return_no_answer,
                 max_seq_len=max_seq_len,
                 use_auth_token=use_auth_token,
-                return_table_cell=return_table_cell,
             )
         else:
             logger.error(
@@ -258,7 +241,6 @@ def __init__(
         tokenizer: Optional[str] = None,
         max_seq_len: int = 256,
         use_auth_token: Optional[Union[str, bool]] = None,
-        return_table_cell: bool = False,
     ):
         self.model = TapasForQuestionAnswering.from_pretrained(
             model_name_or_path, revision=model_version, use_auth_token=use_auth_token
@@ -273,14 +255,13 @@ def __init__(
             )
         self.max_seq_len = max_seq_len
         self.device = device
-        self.pipeline = _TableQuestionAnsweringPipeline(
+        self.pipeline = _TableQuestionAnsweringPipeline(  # type: ignore
             task="table-question-answering",
             model=self.model,
             tokenizer=self.tokenizer,
             framework="pt",
             batch_size=1,  # batch_size of 1 only works currently b/c of issue with HuggingFace pipeline logic and the return type of TableQuestionAnsweringPipeline._forward
             device=self.device,
-            return_table_cell=return_table_cell,
         )
 
     def predict(self, query: str, documents: List[Document], top_k: int) -> Dict:
@@ -322,10 +303,6 @@ def predict_batch(self, queries: List[str], documents: List[List[Document]], top
 class _TableQuestionAnsweringPipeline(TableQuestionAnsweringPipeline):  # pylint: disable=useless-object-inheritance
     """Modified from transformers TableQuestionAnsweringPipeline.postprocess to return Haystack Answer objects."""
 
-    def __init__(self, *args, return_table_cell: bool = False, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.return_table_cell = return_table_cell
-
     def _calculate_answer_score(
         self, logits: "torch.Tensor", inputs: Dict, answer_coordinates: List[List[Tuple[int, int]]]
     ) -> List[Optional[np.ndarray]]:
@@ -440,10 +417,7 @@ def postprocess(self, model_outputs):
                     else:
                         answer_str = self._aggregate_answers(aggregator, cells)
                     current_score = answer_scores[index]
-                    if self.return_table_cell:
-                        answer_offsets = _calculate_answer_offsets(ans_coordinates_per_table)
-                    else:
-                        answer_offsets = _calculate_answer_offsets_span(ans_coordinates_per_table, string_table)
+                    answer_offsets = _calculate_answer_offsets(ans_coordinates_per_table)
                     answer = Answer(
                         answer=answer_str,
                         type="extractive",
@@ -474,7 +448,6 @@ def __init__(
         return_no_answer: bool = False,
         max_seq_len: int = 256,
         use_auth_token: Optional[Union[str, bool]] = None,
-        return_table_cell: bool = False,
     ):
         self.model = self._TapasForScoredQA.from_pretrained(  # type: ignore
             model_name_or_path, revision=model_version, use_auth_token=use_auth_token
@@ -487,7 +460,6 @@ def __init__(
         self.device = device
         self.top_k_per_candidate = top_k_per_candidate
         self.return_no_answer = return_no_answer
-        self.return_table_cell = return_table_cell
 
     def _predict_tapas_scored(self, inputs: "BatchEncoding", document: Document) -> Tuple[List[Answer], float]:
         orig_table: pd.DataFrame = document.content
@@ -569,10 +541,7 @@ def _predict_tapas_scored(self, inputs: "BatchEncoding", document: Document) ->
             current_answer_span = possible_answer_spans[answer_span_idx]
             answer_str = string_table.iat[current_answer_span[:2]]
             answer_offsets: Union[List[Span], List[TableCell]]
-            if self.return_table_cell:
-                answer_offsets = _calculate_answer_offsets([current_answer_span[:2]])
-            else:
-                answer_offsets = _calculate_answer_offsets_span([current_answer_span[:2]], string_table)
+            answer_offsets = _calculate_answer_offsets([current_answer_span[:2]])
             # As the general table score is more important for the final score, it is double weighted.
             current_score = ((2 * table_relevancy_prob) + span_logits_softmax[0, answer_span_idx].item()) / 3
 
@@ -609,12 +578,8 @@ def predict(self, query: str, documents: List[Document], top_k: int) -> Dict:
                 no_answer_score = current_no_answer_score
 
         if self.return_no_answer:
-            if self.return_table_cell:
-                offsets_in_context = None
-                offsets_in_document = None
-            else:
-                offsets_in_context = [Span(start=0, end=0)]
-                offsets_in_document = [Span(start=0, end=0)]
+            offsets_in_context = None
+            offsets_in_document = None
             answers.append(
                 Answer(
                     answer="",
@@ -690,7 +655,6 @@ def __init__(
         top_k: int = 10,
         max_seq_len: int = 256,
         use_auth_token: Optional[Union[str, bool]] = None,
-        return_table_cell: bool = False,
     ):
         """
         Load an RCI model from Transformers.
@@ -719,24 +683,10 @@ def __init__(
                                 `transformers-cli login` (stored in ~/.huggingface) will be used.
                                 Additional information can be found here
                                 https://huggingface.co/transformers/main_classes/model.html#transformers.PreTrainedModel.from_pretrained
-        :param return_table_cell: Whether to return the offsets (`offsets_in_document` and `offsets_in_context`) that indicate
-                                  the cells that answer the question using the TableCell schema. The TableCell schema returns the row
-                                  and column indices of the table cells selected in the Answer. Otherwise, the offsets
-                                  are returned as Span objects which are start and end indices when counting through the
-                                  table in a linear fashion, which means the first cell is top left and the last cell is bottom right.
         """
         torch_and_transformers_import.check()
         super().__init__()
 
-        if not return_table_cell:
-            logger.warning(
-                "The support for returning offsets in answer predictions in a linear fashion is being deprecated."
-                " Set return_table_cell=True to use the new offsets format which returns the row and column indices"
-                " of the table cells selected in the answer."
-                " In the future, return_table_cell=True will become default and return_table_cell=False will no "
-                " longer be supported."
-            )
-
         self.devices, _ = initialize_device_settings(use_cuda=use_gpu, multi_gpu=False)
         if len(self.devices) > 1:
             logger.warning(
@@ -779,7 +729,6 @@ def __init__(
         self.top_k = top_k
         self.max_seq_len = max_seq_len
         self.return_no_answers = False
-        self.return_table_cell = return_table_cell
 
     def predict(self, query: str, documents: List[Document], top_k: Optional[int] = None) -> Dict:
         """
@@ -846,10 +795,7 @@ def predict(self, query: str, documents: List[Document], top_k: Optional[int] =
 
                     answer_str = string_table.iloc[row_idx, col_idx]
                     answer_offsets: Union[List[Span], List[TableCell]]
-                    if self.return_table_cell:
-                        answer_offsets = [TableCell(row=row_idx, col=col_idx)]
-                    else:
-                        answer_offsets = [self._calculate_answer_offsets_span(row_idx, col_idx, string_table)]
+                    answer_offsets = [TableCell(row=row_idx, col=col_idx)]
                     current_answers.append(
                         Answer(
                             answer=answer_str,
@@ -892,13 +838,6 @@ def _create_row_column_representations(table: pd.DataFrame) -> Tuple[List[str],
 
         return row_reps, column_reps
 
-    @staticmethod
-    def _calculate_answer_offsets_span(row_idx, column_index, table) -> Span:
-        _, n_columns = table.shape
-        answer_cell_offset = (row_idx * n_columns) + column_index
-
-        return Span(start=answer_cell_offset, end=answer_cell_offset + 1)
-
     def predict_batch(
         self,
         queries: List[str],
@@ -942,21 +881,6 @@ def _calculate_answer_offsets(answer_coordinates: List[Tuple[int, int]]) -> List
     return answer_offsets
 
 
-def _calculate_answer_offsets_span(answer_coordinates: List[Tuple[int, int]], table: pd.DataFrame) -> List[Span]:
-    """
-    Calculates the answer cell offsets of the linearized table based on the answer cell coordinates.
-
-    :param answer_coordinates: List of answer coordinates.
-    :param table: Table containing the answers in answer coordinates.
-    """
-    answer_offsets = []
-    _, n_columns = table.shape
-    for coord in answer_coordinates:
-        answer_cell_offset = (coord[0] * n_columns) + coord[1]
-        answer_offsets.append(Span(start=answer_cell_offset, end=answer_cell_offset + 1))
-    return answer_offsets
-
-
 def _check_documents(documents: List[Document]) -> List[Document]:
     """
     Check that the content type of all `documents` is of type 'table' otherwise remove that document from the list.


Test patch:
diff --git a/test/nodes/test_table_reader.py b/test/nodes/test_table_reader.py
--- a/test/nodes/test_table_reader.py
+++ b/test/nodes/test_table_reader.py
@@ -6,7 +6,7 @@
 from haystack.schema import Document, Answer, Span, TableCell
 from haystack.pipelines.base import Pipeline
 
-from haystack.nodes.reader.table import _calculate_answer_offsets_span, _calculate_answer_offsets
+from haystack.nodes.reader.table import _calculate_answer_offsets
 
 
 @pytest.fixture
@@ -40,12 +40,6 @@ def table_doc3():
     return Document(content=pd.DataFrame(data), content_type="table", id="doc3")
 
 
-@pytest.mark.unit
-def test_calculate_answer_offsets_span(table_doc1):
-    offsets_span = _calculate_answer_offsets_span(answer_coordinates=[(0, 1), (1, 3)], table=table_doc1.content)
-    assert offsets_span == [Span(start=1, end=2), Span(start=7, end=8)]
-
-
 @pytest.mark.unit
 def test_calculate_answer_offsets_table_cell(table_doc1):
     offsets_span = _calculate_answer_offsets(answer_coordinates=[(0, 1), (1, 3)])