Repository: huggingface/transformers
Pull request: fix document qa bf16 pipeline

This PR enables low-precision document-question-answering pipeline, the error can be reproduced by
```python
from transformers import pipeline
from PIL import Image
import torch

pipe = pipeline("document-question-answering", model="impira/layoutlm-document-qa", torch_dtype=torch.bfloat16)

question = "What is the purchase amount?"
image = Image.open("document.png")

ans = pipe(image=image, question=question)
print(ans)
```

```
Traceback (most recent call last):
  File "/home/jiqingfe/jiqing_hf/HuggingFace/tests/workloads/test_qa.py", line 10, in <module>
    ans = pipe(image=image, question=question)
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/document_question_answering.py", line 278, in __call__
    return super().__call__(inputs, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py", line 1354, in __call__
    return next(
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/pt_utils.py", line 125, in __next__
    processed = self.infer(item, **self.params)
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/document_question_answering.py", line 459, in postprocess
    answers = self.postprocess_extractive_qa(model_outputs, top_k=top_k, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/document_question_answering.py", line 488, in postprocess_extractive_qa
    starts, ends, scores, min_null_score = select_starts_ends(
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/question_answering.py", line 130, in select_starts_ends
    start = np.where(undesired_tokens_mask, -10000.0, start)
  File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 1216, in __array__
    return self.numpy()
TypeError: Got unsupported ScalarType BFloat16
```

Changed files:
filename: src/transformers/pipelines/document_question_answering.py
patch: @@ -485,6 +485,11 @@ def postprocess_extractive_qa(
         for output in model_outputs:
             words = output["words"]
 
+            if self.framework == "pt" and output["start_logits"].dtype in (torch.bfloat16, torch.float16):
+                output["start_logits"] = output["start_logits"].float()
+            if self.framework == "pt" and output["end_logits"].dtype in (torch.bfloat16, torch.float16):
+                output["end_logits"] = output["end_logits"].float()
+
             starts, ends, scores, min_null_score = select_starts_ends(
                 start=output["start_logits"],
                 end=output["end_logits"],

filename: tests/pipelines/test_pipelines_document_question_answering.py
patch: @@ -14,7 +14,12 @@
 
 import unittest
 
-from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoTokenizer, is_vision_available
+from transformers import (
+    MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
+    AutoTokenizer,
+    is_torch_available,
+    is_vision_available,
+)
 from transformers.pipelines import DocumentQuestionAnsweringPipeline, pipeline
 from transformers.pipelines.document_question_answering import apply_tesseract
 from transformers.testing_utils import (
@@ -24,13 +29,17 @@
     require_pytesseract,
     require_tf,
     require_torch,
+    require_torch_bf16,
     require_vision,
     slow,
 )
 
 from .test_pipelines_common import ANY
 
 
+if is_torch_available():
+    import torch
+
 if is_vision_available():
     from PIL import Image
 
@@ -145,6 +154,42 @@ def test_small_model_pt(self):
         outputs = dqa_pipeline(image=image, question=question, words=words, boxes=boxes, top_k=2)
         self.assertEqual(outputs, [])
 
+    @require_torch
+    @require_torch_bf16
+    @require_detectron2
+    @require_pytesseract
+    def test_small_model_pt_bf16(self):
+        dqa_pipeline = pipeline(
+            "document-question-answering",
+            model="hf-internal-testing/tiny-random-layoutlmv2-for-dqa-test",
+            torch_dtype=torch.bfloat16,
+        )
+        image = INVOICE_URL
+        question = "How many cats are there?"
+
+        expected_output = [
+            {"score": 0.0001, "answer": "oy 2312/2019", "start": 38, "end": 39},
+            {"score": 0.0001, "answer": "oy 2312/2019 DUE", "start": 38, "end": 40},
+        ]
+        outputs = dqa_pipeline(image=image, question=question, top_k=2)
+        self.assertEqual(nested_simplify(outputs, decimals=4), expected_output)
+
+        outputs = dqa_pipeline({"image": image, "question": question}, top_k=2)
+        self.assertEqual(nested_simplify(outputs, decimals=4), expected_output)
+
+        # This image does not detect ANY text in it, meaning layoutlmv2 should fail.
+        # Empty answer probably
+        image = "./tests/fixtures/tests_samples/COCO/000000039769.png"
+        outputs = dqa_pipeline(image=image, question=question, top_k=2)
+        self.assertEqual(outputs, [])
+
+        # We can optionnally pass directly the words and bounding boxes
+        image = "./tests/fixtures/tests_samples/COCO/000000039769.png"
+        words = []
+        boxes = []
+        outputs = dqa_pipeline(image=image, question=question, words=words, boxes=boxes, top_k=2)
+        self.assertEqual(outputs, [])
+
     # 	 TODO: Enable this once hf-internal-testing/tiny-random-donut is implemented
     #    @require_torch
     #    def test_small_model_pt_donut(self):