Ankur Goyal
commited on
Commit
•
73a21c8
1
Parent(s):
57b0bdd
Initial commit
Browse files- README.md +12 -0
- config.json +37 -0
- configuration_layoutlm.py +3 -0
- merges.txt +0 -0
- modeling_layoutlm.py +147 -0
- pipeline_document_question_answering.py +375 -0
- pytorch_model.bin +3 -0
- qa_helpers.py +132 -0
- special_tokens_map.json +1 -0
- tokenizer.json +0 -0
- tokenizer_config.json +1 -0
- vocab.json +0 -0
README.md
CHANGED
@@ -1,3 +1,15 @@
|
|
1 |
---
|
|
|
|
|
2 |
license: mit
|
3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
---
|
2 |
+
language: en
|
3 |
+
thumbnail: https://uploads-ssl.webflow.com/5e3898dff507782a6580d710/614a23fcd8d4f7434c765ab9_logo.png
|
4 |
license: mit
|
5 |
---
|
6 |
+
|
7 |
+
# LayoutLM for Visual Question Answering
|
8 |
+
|
9 |
+
This is a fine-tuned version of the multi-modal [LayoutLM](https://aka.ms/layoutlm) model for the task of question answering on documents. It has been fine-tuned on
|
10 |
+
|
11 |
+
## Model details
|
12 |
+
|
13 |
+
The LayoutLM model was developed at Microsoft ([paper](https://arxiv.org/abs/1912.13318)) as a general purpose tool for understanding documents. This model is a fine-tuned checkpoint of [LayoutLM-Base-Cased](https://huggingface.co/microsoft/layoutlm-base-uncased), using both the [SQuAD2.0](https://huggingface.co/datasets/squad_v2) and [DocVQA](https://www.docvqa.org/) datasets.
|
14 |
+
|
15 |
+
## Getting started with the model
|
config.json
ADDED
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"attention_probs_dropout_prob": 0.1,
|
3 |
+
"architectures": [
|
4 |
+
"LayoutLMForQuestionAnswering"
|
5 |
+
],
|
6 |
+
"auto_map": {
|
7 |
+
"AutoConfig": "configuration_layoutlm.LayoutLMConfig",
|
8 |
+
"AutoModelForQuestionAnswering": "modeling_layoutlm.LayoutLMForQuestionAnswering"
|
9 |
+
},
|
10 |
+
"custom_pipelines": {
|
11 |
+
"document-question-answering": {
|
12 |
+
"impl": "pipeline_document_question_answering.DocumentQuestionAnsweringPipeline",
|
13 |
+
"pt": "AutoModelForQuestionAnswering"
|
14 |
+
}
|
15 |
+
},
|
16 |
+
"bos_token_id": 0,
|
17 |
+
"eos_token_id": 2,
|
18 |
+
"gradient_checkpointing": false,
|
19 |
+
"hidden_act": "gelu",
|
20 |
+
"hidden_dropout_prob": 0.1,
|
21 |
+
"hidden_size": 768,
|
22 |
+
"initializer_range": 0.02,
|
23 |
+
"intermediate_size": 3072,
|
24 |
+
"layer_norm_eps": 1e-05,
|
25 |
+
"max_2d_position_embeddings": 1024,
|
26 |
+
"max_position_embeddings": 514,
|
27 |
+
"model_type": "layoutlm",
|
28 |
+
"num_attention_heads": 12,
|
29 |
+
"num_hidden_layers": 12,
|
30 |
+
"pad_token_id": 1,
|
31 |
+
"position_embedding_type": "absolute",
|
32 |
+
"tokenizer_class": "RobertaTokenizer",
|
33 |
+
"transformers_version": "4.6.1",
|
34 |
+
"type_vocab_size": 1,
|
35 |
+
"use_cache": true,
|
36 |
+
"vocab_size": 50265
|
37 |
+
}
|
configuration_layoutlm.py
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
# This model just uses the existing LayoutLMConfig which is just imported
|
2 |
+
# as a thin wrapper
|
3 |
+
from transformers.models.layoutlm.configuration_layoutlm import LayoutLMConfig
|
merges.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
modeling_layoutlm.py
ADDED
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# NOTE: This code is currently under review for inclusion in the main
|
2 |
+
# huggingface/transformers repository:
|
3 |
+
# https://github.com/huggingface/transformers/pull/18407
|
4 |
+
""" PyTorch LayoutLM model."""
|
5 |
+
|
6 |
+
|
7 |
+
import math
|
8 |
+
from typing import Optional, Tuple, Union
|
9 |
+
|
10 |
+
import torch
|
11 |
+
from torch import nn
|
12 |
+
from torch.nn import CrossEntropyLoss
|
13 |
+
|
14 |
+
from transformers.modeling_outputs import QuestionAnsweringModelOutput
|
15 |
+
from transformers.models.layoutlm import LayoutLMModel, LayoutLMPreTrainedModel
|
16 |
+
|
17 |
+
|
18 |
+
class LayoutLMForQuestionAnswering(LayoutLMPreTrainedModel):
|
19 |
+
def __init__(self, config, has_visual_segment_embedding=True):
|
20 |
+
super().__init__(config)
|
21 |
+
self.num_labels = config.num_labels
|
22 |
+
|
23 |
+
self.layoutlm = LayoutLMModel(config)
|
24 |
+
self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
|
25 |
+
|
26 |
+
# Initialize weights and apply final processing
|
27 |
+
self.post_init()
|
28 |
+
|
29 |
+
def get_input_embeddings(self):
|
30 |
+
return self.layoutlm.embeddings.word_embeddings
|
31 |
+
|
32 |
+
def forward(
|
33 |
+
self,
|
34 |
+
input_ids: Optional[torch.LongTensor] = None,
|
35 |
+
bbox: Optional[torch.LongTensor] = None,
|
36 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
37 |
+
token_type_ids: Optional[torch.LongTensor] = None,
|
38 |
+
position_ids: Optional[torch.LongTensor] = None,
|
39 |
+
head_mask: Optional[torch.FloatTensor] = None,
|
40 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
41 |
+
start_positions: Optional[torch.LongTensor] = None,
|
42 |
+
end_positions: Optional[torch.LongTensor] = None,
|
43 |
+
output_attentions: Optional[bool] = None,
|
44 |
+
output_hidden_states: Optional[bool] = None,
|
45 |
+
return_dict: Optional[bool] = None,
|
46 |
+
) -> Union[Tuple, QuestionAnsweringModelOutput]:
|
47 |
+
r"""
|
48 |
+
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
49 |
+
Labels for position (index) of the start of the labelled span for computing the token classification loss.
|
50 |
+
Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
|
51 |
+
are not taken into account for computing the loss.
|
52 |
+
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
53 |
+
Labels for position (index) of the end of the labelled span for computing the token classification loss.
|
54 |
+
Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
|
55 |
+
are not taken into account for computing the loss.
|
56 |
+
|
57 |
+
Returns:
|
58 |
+
|
59 |
+
Example:
|
60 |
+
|
61 |
+
In this example below, we give the LayoutLMv2 model an image (of texts) and ask it a question. It will give us
|
62 |
+
a prediction of what it thinks the answer is (the span of the answer within the texts parsed from the image).
|
63 |
+
|
64 |
+
```python
|
65 |
+
>>> from transformers import AutoTokenizer, LayoutLMForQuestionAnswering
|
66 |
+
>>> from datasets import load_dataset
|
67 |
+
>>> import torch
|
68 |
+
|
69 |
+
>>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased", add_prefix_space=True)
|
70 |
+
>>> model = LayoutLMForQuestionAnswering.from_pretrained("microsoft/layoutlm-base-uncased")
|
71 |
+
|
72 |
+
>>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train")
|
73 |
+
>>> example = dataset[0]
|
74 |
+
>>> question = "what's his name?"
|
75 |
+
>>> words = example["tokens"]
|
76 |
+
>>> boxes = example["bboxes"]
|
77 |
+
|
78 |
+
>>> encoding = tokenizer(
|
79 |
+
... question.split(), words, is_split_into_words=True, return_token_type_ids=True, return_tensors="pt"
|
80 |
+
... )
|
81 |
+
>>> bbox = []
|
82 |
+
>>> for i, s, w in zip(encoding.input_ids[0], encoding.sequence_ids(0), encoding.word_ids(0)):
|
83 |
+
... if s == 1:
|
84 |
+
... bbox.append(boxes[w])
|
85 |
+
... elif i == tokenizer.sep_token_id:
|
86 |
+
... bbox.append([1000] * 4)
|
87 |
+
... else:
|
88 |
+
... bbox.append([0] * 4)
|
89 |
+
>>> encoding["bbox"] = torch.tensor([bbox])
|
90 |
+
|
91 |
+
>>> outputs = model(**encoding)
|
92 |
+
>>> loss = outputs.loss
|
93 |
+
>>> start_scores = outputs.start_logits
|
94 |
+
>>> end_scores = outputs.end_logits
|
95 |
+
```
|
96 |
+
"""
|
97 |
+
|
98 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
99 |
+
|
100 |
+
outputs = self.layoutlm(
|
101 |
+
input_ids=input_ids,
|
102 |
+
bbox=bbox,
|
103 |
+
attention_mask=attention_mask,
|
104 |
+
token_type_ids=token_type_ids,
|
105 |
+
position_ids=position_ids,
|
106 |
+
head_mask=head_mask,
|
107 |
+
inputs_embeds=inputs_embeds,
|
108 |
+
output_attentions=output_attentions,
|
109 |
+
output_hidden_states=output_hidden_states,
|
110 |
+
return_dict=return_dict,
|
111 |
+
)
|
112 |
+
|
113 |
+
sequence_output = outputs[0]
|
114 |
+
|
115 |
+
logits = self.qa_outputs(sequence_output)
|
116 |
+
start_logits, end_logits = logits.split(1, dim=-1)
|
117 |
+
start_logits = start_logits.squeeze(-1).contiguous()
|
118 |
+
end_logits = end_logits.squeeze(-1).contiguous()
|
119 |
+
|
120 |
+
total_loss = None
|
121 |
+
if start_positions is not None and end_positions is not None:
|
122 |
+
# If we are on multi-GPU, split add a dimension
|
123 |
+
if len(start_positions.size()) > 1:
|
124 |
+
start_positions = start_positions.squeeze(-1)
|
125 |
+
if len(end_positions.size()) > 1:
|
126 |
+
end_positions = end_positions.squeeze(-1)
|
127 |
+
# sometimes the start/end positions are outside our model inputs, we ignore these terms
|
128 |
+
ignored_index = start_logits.size(1)
|
129 |
+
start_positions = start_positions.clamp(0, ignored_index)
|
130 |
+
end_positions = end_positions.clamp(0, ignored_index)
|
131 |
+
|
132 |
+
loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
|
133 |
+
start_loss = loss_fct(start_logits, start_positions)
|
134 |
+
end_loss = loss_fct(end_logits, end_positions)
|
135 |
+
total_loss = (start_loss + end_loss) / 2
|
136 |
+
|
137 |
+
if not return_dict:
|
138 |
+
output = (start_logits, end_logits) + outputs[2:]
|
139 |
+
return ((total_loss,) + output) if total_loss is not None else output
|
140 |
+
|
141 |
+
return QuestionAnsweringModelOutput(
|
142 |
+
loss=total_loss,
|
143 |
+
start_logits=start_logits,
|
144 |
+
end_logits=end_logits,
|
145 |
+
hidden_states=outputs.hidden_states,
|
146 |
+
attentions=outputs.attentions,
|
147 |
+
)
|
pipeline_document_question_answering.py
ADDED
@@ -0,0 +1,375 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# NOTE: This code is currently under review for inclusion in the main
|
2 |
+
# huggingface/transformers repository:
|
3 |
+
# https://github.com/huggingface/transformers/pull/18414
|
4 |
+
from typing import List, Optional, Tuple, Union
|
5 |
+
|
6 |
+
import numpy as np
|
7 |
+
|
8 |
+
from transformers.utils import add_end_docstrings, is_torch_available, logging
|
9 |
+
from transformers.pipelines.base import PIPELINE_INIT_ARGS, Pipeline
|
10 |
+
from .qa_helpers import select_starts_ends, Image, load_image, VISION_LOADED, pytesseract, TESSERACT_LOADED
|
11 |
+
|
12 |
+
|
13 |
+
if is_torch_available():
|
14 |
+
import torch
|
15 |
+
|
16 |
+
# We do not perform the check in this version of the pipeline code
|
17 |
+
# from transformers.models.auto.modeling_auto import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING
|
18 |
+
|
19 |
+
logger = logging.get_logger(__name__)
|
20 |
+
|
21 |
+
|
22 |
+
# normalize_bbox() and apply_tesseract() are derived from apply_tesseract in models/layoutlmv3/feature_extraction_layoutlmv3.py.
|
23 |
+
# However, because the pipeline may evolve from what layoutlmv3 currently does, it's copied (vs. imported) to avoid creating an
|
24 |
+
# unecessary dependency.
|
25 |
+
def normalize_box(box, width, height):
|
26 |
+
return [
|
27 |
+
int(1000 * (box[0] / width)),
|
28 |
+
int(1000 * (box[1] / height)),
|
29 |
+
int(1000 * (box[2] / width)),
|
30 |
+
int(1000 * (box[3] / height)),
|
31 |
+
]
|
32 |
+
|
33 |
+
|
34 |
+
def apply_tesseract(image: "Image.Image", lang: Optional[str], tesseract_config: Optional[str]):
|
35 |
+
"""Applies Tesseract OCR on a document image, and returns recognized words + normalized bounding boxes."""
|
36 |
+
# apply OCR
|
37 |
+
data = pytesseract.image_to_data(image, lang=lang, output_type="dict", config=tesseract_config)
|
38 |
+
words, left, top, width, height = data["text"], data["left"], data["top"], data["width"], data["height"]
|
39 |
+
|
40 |
+
# filter empty words and corresponding coordinates
|
41 |
+
irrelevant_indices = [idx for idx, word in enumerate(words) if not word.strip()]
|
42 |
+
words = [word for idx, word in enumerate(words) if idx not in irrelevant_indices]
|
43 |
+
left = [coord for idx, coord in enumerate(left) if idx not in irrelevant_indices]
|
44 |
+
top = [coord for idx, coord in enumerate(top) if idx not in irrelevant_indices]
|
45 |
+
width = [coord for idx, coord in enumerate(width) if idx not in irrelevant_indices]
|
46 |
+
height = [coord for idx, coord in enumerate(height) if idx not in irrelevant_indices]
|
47 |
+
|
48 |
+
# turn coordinates into (left, top, left+width, top+height) format
|
49 |
+
actual_boxes = []
|
50 |
+
for x, y, w, h in zip(left, top, width, height):
|
51 |
+
actual_box = [x, y, x + w, y + h]
|
52 |
+
actual_boxes.append(actual_box)
|
53 |
+
|
54 |
+
image_width, image_height = image.size
|
55 |
+
|
56 |
+
# finally, normalize the bounding boxes
|
57 |
+
normalized_boxes = []
|
58 |
+
for box in actual_boxes:
|
59 |
+
normalized_boxes.append(normalize_box(box, image_width, image_height))
|
60 |
+
|
61 |
+
assert len(words) == len(normalized_boxes), "Not as many words as there are bounding boxes"
|
62 |
+
|
63 |
+
return words, normalized_boxes
|
64 |
+
|
65 |
+
|
66 |
+
@add_end_docstrings(PIPELINE_INIT_ARGS)
|
67 |
+
class DocumentQuestionAnsweringPipeline(Pipeline):
|
68 |
+
# TODO: Update task_summary docs to include an example with document QA and then update the first sentence
|
69 |
+
"""
|
70 |
+
Document Question Answering pipeline using any `AutoModelForDocumentQuestionAnswering`. See the [question answering
|
71 |
+
examples](../task_summary#question-answering) for more information.
|
72 |
+
|
73 |
+
This document question answering pipeline can currently be loaded from [`pipeline`] using the following task
|
74 |
+
identifier: `"document-question-answering"`.
|
75 |
+
|
76 |
+
The models that this pipeline can use are models that have been fine-tuned on a document question answering task.
|
77 |
+
See the up-to-date list of available models on
|
78 |
+
[huggingface.co/models](https://huggingface.co/models?filter=document-question-answering).
|
79 |
+
"""
|
80 |
+
|
81 |
+
def __init__(self, *args, **kwargs):
|
82 |
+
super().__init__(*args, **kwargs)
|
83 |
+
# self.check_model_type(MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING)
|
84 |
+
|
85 |
+
def _sanitize_parameters(
|
86 |
+
self,
|
87 |
+
padding=None,
|
88 |
+
doc_stride=None,
|
89 |
+
max_question_len=None,
|
90 |
+
lang: Optional[str] = None,
|
91 |
+
tesseract_config: Optional[str] = None,
|
92 |
+
max_answer_len=None,
|
93 |
+
max_seq_len=None,
|
94 |
+
top_k=None,
|
95 |
+
handle_impossible_answer=None,
|
96 |
+
**kwargs,
|
97 |
+
):
|
98 |
+
preprocess_params, postprocess_params = {}, {}
|
99 |
+
if padding is not None:
|
100 |
+
preprocess_params["padding"] = padding
|
101 |
+
if doc_stride is not None:
|
102 |
+
preprocess_params["doc_stride"] = doc_stride
|
103 |
+
if max_question_len is not None:
|
104 |
+
preprocess_params["max_question_len"] = max_question_len
|
105 |
+
if max_seq_len is not None:
|
106 |
+
preprocess_params["max_seq_len"] = max_seq_len
|
107 |
+
if lang is not None:
|
108 |
+
preprocess_params["lang"] = lang
|
109 |
+
if tesseract_config is not None:
|
110 |
+
preprocess_params["tesseract_config"] = tesseract_config
|
111 |
+
|
112 |
+
if top_k is not None:
|
113 |
+
if top_k < 1:
|
114 |
+
raise ValueError(f"top_k parameter should be >= 1 (got {top_k})")
|
115 |
+
postprocess_params["top_k"] = top_k
|
116 |
+
if max_answer_len is not None:
|
117 |
+
if max_answer_len < 1:
|
118 |
+
raise ValueError(f"max_answer_len parameter should be >= 1 (got {max_answer_len}")
|
119 |
+
postprocess_params["max_answer_len"] = max_answer_len
|
120 |
+
if handle_impossible_answer is not None:
|
121 |
+
postprocess_params["handle_impossible_answer"] = handle_impossible_answer
|
122 |
+
|
123 |
+
return preprocess_params, {}, postprocess_params
|
124 |
+
|
125 |
+
def __call__(
|
126 |
+
self,
|
127 |
+
image: Union["Image.Image", str],
|
128 |
+
question: Optional[str] = None,
|
129 |
+
word_boxes: Tuple[str, List[float]] = None,
|
130 |
+
**kwargs,
|
131 |
+
):
|
132 |
+
"""
|
133 |
+
Answer the question(s) given as inputs by using the document(s). A document is defined as an image and an
|
134 |
+
optional list of (word, box) tuples which represent the text in the document. If the `word_boxes` are not
|
135 |
+
provided, it will use the Tesseract OCR engine (if available) to extract the words and boxes automatically.
|
136 |
+
|
137 |
+
You can invoke the pipeline several ways:
|
138 |
+
|
139 |
+
- `pipeline(image=image, question=question)`
|
140 |
+
- `pipeline(image=image, question=question, word_boxes=word_boxes)`
|
141 |
+
- `pipeline([{"image": image, "question": question}])`
|
142 |
+
- `pipeline([{"image": image, "question": question, "word_boxes": word_boxes}])`
|
143 |
+
|
144 |
+
Args:
|
145 |
+
image (`str` or `PIL.Image`):
|
146 |
+
The pipeline handles three types of images:
|
147 |
+
|
148 |
+
- A string containing a http link pointing to an image
|
149 |
+
- A string containing a local path to an image
|
150 |
+
- An image loaded in PIL directly
|
151 |
+
|
152 |
+
The pipeline accepts either a single image or a batch of images. If given a single image, it can be
|
153 |
+
broadcasted to multiple questions.
|
154 |
+
question (`str`):
|
155 |
+
A question to ask of the document.
|
156 |
+
word_boxes (`List[str, Tuple[float, float, float, float]]`, *optional*):
|
157 |
+
A list of words and bounding boxes (normalized 0->1000). If you provide this optional input, then the
|
158 |
+
pipeline will use these words and boxes instead of running OCR on the image to derive them. This allows
|
159 |
+
you to reuse OCR'd results across many invocations of the pipeline without having to re-run it each
|
160 |
+
time.
|
161 |
+
top_k (`int`, *optional*, defaults to 1):
|
162 |
+
The number of answers to return (will be chosen by order of likelihood). Note that we return less than
|
163 |
+
top_k answers if there are not enough options available within the context.
|
164 |
+
doc_stride (`int`, *optional*, defaults to 128):
|
165 |
+
If the words in the document are too long to fit with the question for the model, it will be split in
|
166 |
+
several chunks with some overlap. This argument controls the size of that overlap.
|
167 |
+
max_answer_len (`int`, *optional*, defaults to 15):
|
168 |
+
The maximum length of predicted answers (e.g., only answers with a shorter length are considered).
|
169 |
+
max_seq_len (`int`, *optional*, defaults to 384):
|
170 |
+
The maximum length of the total sentence (context + question) in tokens of each chunk passed to the
|
171 |
+
model. The context will be split in several chunks (using `doc_stride` as overlap) if needed.
|
172 |
+
max_question_len (`int`, *optional*, defaults to 64):
|
173 |
+
The maximum length of the question after tokenization. It will be truncated if needed.
|
174 |
+
handle_impossible_answer (`bool`, *optional*, defaults to `False`):
|
175 |
+
Whether or not we accept impossible as an answer.
|
176 |
+
lang (`str`, *optional*):
|
177 |
+
Language to use while running OCR. Defaults to english.
|
178 |
+
tesseract_config (`str`, *optional*):
|
179 |
+
Additional flags to pass to tesseract while running OCR.
|
180 |
+
|
181 |
+
Return:
|
182 |
+
A `dict` or a list of `dict`: Each result comes as a dictionary with the following keys:
|
183 |
+
|
184 |
+
- **score** (`float`) -- The probability associated to the answer.
|
185 |
+
- **start** (`int`) -- The start word index of the answer (in the OCR'd version of the input or provided
|
186 |
+
`word_boxes`).
|
187 |
+
- **end** (`int`) -- The end word index of the answer (in the OCR'd version of the input or provided
|
188 |
+
`word_boxes`).
|
189 |
+
- **answer** (`str`) -- The answer to the question.
|
190 |
+
"""
|
191 |
+
if isinstance(question, str):
|
192 |
+
inputs = {"question": question, "image": image, "word_boxes": word_boxes}
|
193 |
+
else:
|
194 |
+
inputs = image
|
195 |
+
return super().__call__(inputs, **kwargs)
|
196 |
+
|
197 |
+
def preprocess(
|
198 |
+
self,
|
199 |
+
input,
|
200 |
+
padding="do_not_pad",
|
201 |
+
doc_stride=None,
|
202 |
+
max_question_len=64,
|
203 |
+
max_seq_len=None,
|
204 |
+
word_boxes: Tuple[str, List[float]] = None,
|
205 |
+
lang=None,
|
206 |
+
tesseract_config="",
|
207 |
+
):
|
208 |
+
# NOTE: This code mirrors the code in question answering and will be implemented in a follow up PR
|
209 |
+
# to support documents with enough tokens that overflow the model's window
|
210 |
+
# if max_seq_len is None:
|
211 |
+
# # TODO: LayoutLM's stride is 512 by default. Is it ok to use that as the min
|
212 |
+
# # instead of 384 (which the QA model uses)?
|
213 |
+
# max_seq_len = min(self.tokenizer.model_max_length, 512)
|
214 |
+
|
215 |
+
if doc_stride is not None:
|
216 |
+
# TODO implement
|
217 |
+
# doc_stride = min(max_seq_len // 2, 128)
|
218 |
+
raise ValueError("Unsupported: striding inputs")
|
219 |
+
|
220 |
+
image = None
|
221 |
+
image_features = {}
|
222 |
+
if "image" in input:
|
223 |
+
if not VISION_LOADED:
|
224 |
+
raise ValueError(
|
225 |
+
"If you provide an image, then the pipeline will run process it with PIL (Pillow), but"
|
226 |
+
" PIL is not available. Install it with pip install Pillow."
|
227 |
+
)
|
228 |
+
image = load_image(input["image"])
|
229 |
+
if self.feature_extractor is not None:
|
230 |
+
image_features.update(self.feature_extractor(images=image, return_tensors=self.framework))
|
231 |
+
|
232 |
+
words, boxes = None, None
|
233 |
+
if "word_boxes" in input:
|
234 |
+
words = [x[0] for x in input["word_boxes"]]
|
235 |
+
boxes = [x[1] for x in input["word_boxes"]]
|
236 |
+
elif "words" in image_features and "boxes" in image_features:
|
237 |
+
words = image_features.pop("words")
|
238 |
+
boxes = image_features.pop("boxes")
|
239 |
+
elif image is not None:
|
240 |
+
if not TESSERACT_LOADED:
|
241 |
+
raise ValueError(
|
242 |
+
"If you provide an image without word_boxes, then the pipeline will run OCR using Tesseract, but"
|
243 |
+
" pytesseract is not available. Install it with pip install pytesseract."
|
244 |
+
)
|
245 |
+
words, boxes = apply_tesseract(image, lang=lang, tesseract_config=tesseract_config)
|
246 |
+
else:
|
247 |
+
raise ValueError(
|
248 |
+
"You must provide an image or word_boxes. If you provide an image, the pipeline will automatically run"
|
249 |
+
" OCR to derive words and boxes"
|
250 |
+
)
|
251 |
+
|
252 |
+
if self.tokenizer.padding_side != "right":
|
253 |
+
raise ValueError(
|
254 |
+
"Document question answering only supports tokenizers whose padding side is 'right', not"
|
255 |
+
f" {self.tokenizer.padding_side}"
|
256 |
+
)
|
257 |
+
|
258 |
+
encoding = self.tokenizer(
|
259 |
+
text=input["question"].split(),
|
260 |
+
text_pair=words,
|
261 |
+
padding=padding,
|
262 |
+
max_length=max_seq_len,
|
263 |
+
stride=doc_stride,
|
264 |
+
return_token_type_ids=True,
|
265 |
+
is_split_into_words=True,
|
266 |
+
return_tensors=self.framework,
|
267 |
+
# TODO: In a future PR, use these feature to handle sequences whose length is longer than
|
268 |
+
# the maximum allowed by the model. Currently, the tokenizer will produce a sequence that
|
269 |
+
# may be too long for the model to handle.
|
270 |
+
# truncation="only_second",
|
271 |
+
# return_overflowing_tokens=True,
|
272 |
+
)
|
273 |
+
encoding.update(image_features)
|
274 |
+
|
275 |
+
# TODO: For now, this should always be num_spans == 1 given the flags we've passed in above, but the
|
276 |
+
# code is written to naturally handle multiple spans at the right time.
|
277 |
+
num_spans = len(encoding["input_ids"])
|
278 |
+
|
279 |
+
# p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer)
|
280 |
+
# We put 0 on the tokens from the context and 1 everywhere else (question and special tokens)
|
281 |
+
# This logic mirrors the logic in the question_answering pipeline
|
282 |
+
p_mask = [[tok != 1 for tok in encoding.sequence_ids(span_id)] for span_id in range(num_spans)]
|
283 |
+
for span_idx in range(num_spans):
|
284 |
+
input_ids_span_idx = encoding["input_ids"][span_idx]
|
285 |
+
# keep the cls_token unmasked (some models use it to indicate unanswerable questions)
|
286 |
+
if self.tokenizer.cls_token_id is not None:
|
287 |
+
cls_indices = np.nonzero(np.array(input_ids_span_idx) == self.tokenizer.cls_token_id)[0]
|
288 |
+
for cls_index in cls_indices:
|
289 |
+
p_mask[span_idx][cls_index] = 0
|
290 |
+
|
291 |
+
# For each span, place a bounding box [0,0,0,0] for question and CLS tokens, [1000,1000,1000,1000]
|
292 |
+
# for SEP tokens, and the word's bounding box for words in the original document.
|
293 |
+
bbox = []
|
294 |
+
for batch_index in range(num_spans):
|
295 |
+
for i, s, w in zip(
|
296 |
+
encoding.input_ids[batch_index],
|
297 |
+
encoding.sequence_ids(batch_index),
|
298 |
+
encoding.word_ids(batch_index),
|
299 |
+
):
|
300 |
+
if s == 1:
|
301 |
+
bbox.append(boxes[w])
|
302 |
+
elif i == self.tokenizer.sep_token_id:
|
303 |
+
bbox.append([1000] * 4)
|
304 |
+
else:
|
305 |
+
bbox.append([0] * 4)
|
306 |
+
|
307 |
+
if self.framework == "tf":
|
308 |
+
raise ValueError("Unsupported: Tensorflow preprocessing for DocumentQuestionAnsweringPipeline")
|
309 |
+
elif self.framework == "pt":
|
310 |
+
encoding["bbox"] = torch.tensor([bbox])
|
311 |
+
|
312 |
+
word_ids = [encoding.word_ids(i) for i in range(num_spans)]
|
313 |
+
|
314 |
+
# TODO This will be necessary when we implement overflow support
|
315 |
+
# encoding.pop("overflow_to_sample_mapping", None)
|
316 |
+
|
317 |
+
return {
|
318 |
+
**encoding,
|
319 |
+
"p_mask": p_mask,
|
320 |
+
"word_ids": word_ids,
|
321 |
+
"words": words,
|
322 |
+
}
|
323 |
+
|
324 |
+
def _forward(self, model_inputs):
|
325 |
+
p_mask = model_inputs.pop("p_mask", None)
|
326 |
+
word_ids = model_inputs.pop("word_ids", None)
|
327 |
+
words = model_inputs.pop("words", None)
|
328 |
+
|
329 |
+
model_outputs = self.model(**model_inputs)
|
330 |
+
|
331 |
+
model_outputs["p_mask"] = p_mask
|
332 |
+
model_outputs["word_ids"] = word_ids
|
333 |
+
model_outputs["words"] = words
|
334 |
+
model_outputs["attention_mask"] = model_inputs["attention_mask"]
|
335 |
+
return model_outputs
|
336 |
+
|
337 |
+
def postprocess(self, model_outputs, top_k=1, handle_impossible_answer=False, max_answer_len=15):
|
338 |
+
min_null_score = 1000000 # large and positive
|
339 |
+
answers = []
|
340 |
+
words = model_outputs["words"]
|
341 |
+
|
342 |
+
# TODO: Currently, we expect the length of model_outputs to be 1, because we do not stride
|
343 |
+
# in the preprocessor code. When we implement that, we'll either need to handle tensors of size
|
344 |
+
# > 1 or use the ChunkPipeline and handle multiple outputs (each of size = 1).
|
345 |
+
starts, ends, scores, min_null_score = select_starts_ends(
|
346 |
+
model_outputs["start_logits"],
|
347 |
+
model_outputs["end_logits"],
|
348 |
+
model_outputs["p_mask"],
|
349 |
+
model_outputs["attention_mask"].numpy() if model_outputs.get("attention_mask", None) is not None else None,
|
350 |
+
min_null_score,
|
351 |
+
top_k,
|
352 |
+
handle_impossible_answer,
|
353 |
+
max_answer_len,
|
354 |
+
)
|
355 |
+
|
356 |
+
word_ids = model_outputs["word_ids"][0]
|
357 |
+
for s, e, score in zip(starts, ends, scores):
|
358 |
+
word_start, word_end = word_ids[s], word_ids[e]
|
359 |
+
if word_start is not None and word_end is not None:
|
360 |
+
answers.append(
|
361 |
+
{
|
362 |
+
"score": score,
|
363 |
+
"answer": " ".join(words[word_start : word_end + 1]),
|
364 |
+
"start": word_start,
|
365 |
+
"end": word_end,
|
366 |
+
}
|
367 |
+
)
|
368 |
+
|
369 |
+
if handle_impossible_answer:
|
370 |
+
answers.append({"score": min_null_score, "answer": "", "start": 0, "end": 0})
|
371 |
+
|
372 |
+
answers = sorted(answers, key=lambda x: x["score"], reverse=True)[:top_k]
|
373 |
+
if len(answers) == 1:
|
374 |
+
return answers[0]
|
375 |
+
return answers
|
pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:8870505d29315260ff57436aab0c66f3a2ddfb2cc7e09a2e368e04e762d0baba
|
3 |
+
size 511244837
|
qa_helpers.py
ADDED
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# NOTE: This code is currently under review for inclusion in the main
|
2 |
+
# huggingface/transformers repository:
|
3 |
+
# https://github.com/huggingface/transformers/pull/18414
|
4 |
+
|
5 |
+
import warnings
|
6 |
+
from collections.abc import Iterable
|
7 |
+
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
|
8 |
+
|
9 |
+
import numpy as np
|
10 |
+
|
11 |
+
from transformers.utils import is_pytesseract_available, is_vision_available
|
12 |
+
|
13 |
+
VISION_LOADED = False
|
14 |
+
if is_vision_available():
|
15 |
+
from PIL import Image
|
16 |
+
|
17 |
+
from transformers.image_utils import load_image
|
18 |
+
VISION_LOADED = True
|
19 |
+
else:
|
20 |
+
Image = None
|
21 |
+
load_image = None
|
22 |
+
|
23 |
+
|
24 |
+
TESSERACT_LOADED = False
|
25 |
+
if is_pytesseract_available():
|
26 |
+
import pytesseract
|
27 |
+
TESSERACT_LOADED = True
|
28 |
+
else:
|
29 |
+
pytesseract = None
|
30 |
+
|
31 |
+
def decode_spans(
|
32 |
+
start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int, undesired_tokens: np.ndarray
|
33 |
+
) -> Tuple:
|
34 |
+
"""
|
35 |
+
Take the output of any `ModelForQuestionAnswering` and will generate probabilities for each span to be the actual
|
36 |
+
answer.
|
37 |
+
|
38 |
+
In addition, it filters out some unwanted/impossible cases like answer len being greater than max_answer_len or
|
39 |
+
answer end position being before the starting position. The method supports output the k-best answer through the
|
40 |
+
topk argument.
|
41 |
+
|
42 |
+
Args:
|
43 |
+
start (`np.ndarray`): Individual start probabilities for each token.
|
44 |
+
end (`np.ndarray`): Individual end probabilities for each token.
|
45 |
+
topk (`int`): Indicates how many possible answer span(s) to extract from the model output.
|
46 |
+
max_answer_len (`int`): Maximum size of the answer to extract from the model's output.
|
47 |
+
undesired_tokens (`np.ndarray`): Mask determining tokens that can be part of the answer
|
48 |
+
"""
|
49 |
+
# Ensure we have batch axis
|
50 |
+
if start.ndim == 1:
|
51 |
+
start = start[None]
|
52 |
+
|
53 |
+
if end.ndim == 1:
|
54 |
+
end = end[None]
|
55 |
+
|
56 |
+
# Compute the score of each tuple(start, end) to be the real answer
|
57 |
+
outer = np.matmul(np.expand_dims(start, -1), np.expand_dims(end, 1))
|
58 |
+
|
59 |
+
# Remove candidate with end < start and end - start > max_answer_len
|
60 |
+
candidates = np.tril(np.triu(outer), max_answer_len - 1)
|
61 |
+
|
62 |
+
# Inspired by Chen & al. (https://github.com/facebookresearch/DrQA)
|
63 |
+
scores_flat = candidates.flatten()
|
64 |
+
if topk == 1:
|
65 |
+
idx_sort = [np.argmax(scores_flat)]
|
66 |
+
elif len(scores_flat) < topk:
|
67 |
+
idx_sort = np.argsort(-scores_flat)
|
68 |
+
else:
|
69 |
+
idx = np.argpartition(-scores_flat, topk)[0:topk]
|
70 |
+
idx_sort = idx[np.argsort(-scores_flat[idx])]
|
71 |
+
|
72 |
+
starts, ends = np.unravel_index(idx_sort, candidates.shape)[1:]
|
73 |
+
desired_spans = np.isin(starts, undesired_tokens.nonzero()) & np.isin(ends, undesired_tokens.nonzero())
|
74 |
+
starts = starts[desired_spans]
|
75 |
+
ends = ends[desired_spans]
|
76 |
+
scores = candidates[0, starts, ends]
|
77 |
+
|
78 |
+
return starts, ends, scores
|
79 |
+
|
80 |
+
|
81 |
+
def select_starts_ends(
|
82 |
+
start,
|
83 |
+
end,
|
84 |
+
p_mask,
|
85 |
+
attention_mask,
|
86 |
+
min_null_score=1000000,
|
87 |
+
top_k=1,
|
88 |
+
handle_impossible_answer=False,
|
89 |
+
max_answer_len=15,
|
90 |
+
):
|
91 |
+
"""
|
92 |
+
Takes the raw output of any `ModelForQuestionAnswering` and first normalizes its outputs and then uses
|
93 |
+
`decode_spans()` to generate probabilities for each span to be the actual answer.
|
94 |
+
|
95 |
+
Args:
|
96 |
+
start (`np.ndarray`): Individual start probabilities for each token.
|
97 |
+
end (`np.ndarray`): Individual end probabilities for each token.
|
98 |
+
p_mask (`np.ndarray`): A mask with 1 for values that cannot be in the answer
|
99 |
+
attention_mask (`np.ndarray`): The attention mask generated by the tokenizer
|
100 |
+
min_null_score(`float`): The minimum null (empty) answer score seen so far.
|
101 |
+
topk (`int`): Indicates how many possible answer span(s) to extract from the model output.
|
102 |
+
handle_impossible_answer(`bool`): Whether to allow null (empty) answers
|
103 |
+
max_answer_len (`int`): Maximum size of the answer to extract from the model's output.
|
104 |
+
"""
|
105 |
+
# Ensure padded tokens & question tokens cannot belong to the set of candidate answers.
|
106 |
+
undesired_tokens = np.abs(np.array(p_mask) - 1)
|
107 |
+
|
108 |
+
if attention_mask is not None:
|
109 |
+
undesired_tokens = undesired_tokens & attention_mask
|
110 |
+
|
111 |
+
# Generate mask
|
112 |
+
undesired_tokens_mask = undesired_tokens == 0.0
|
113 |
+
|
114 |
+
# Make sure non-context indexes in the tensor cannot contribute to the softmax
|
115 |
+
start = np.where(undesired_tokens_mask, -10000.0, start)
|
116 |
+
end = np.where(undesired_tokens_mask, -10000.0, end)
|
117 |
+
|
118 |
+
# Normalize logits and spans to retrieve the answer
|
119 |
+
start = np.exp(start - start.max(axis=-1, keepdims=True))
|
120 |
+
start = start / start.sum()
|
121 |
+
|
122 |
+
end = np.exp(end - end.max(axis=-1, keepdims=True))
|
123 |
+
end = end / end.sum()
|
124 |
+
|
125 |
+
if handle_impossible_answer:
|
126 |
+
min_null_score = min(min_null_score, (start[0, 0] * end[0, 0]).item())
|
127 |
+
|
128 |
+
# Mask CLS
|
129 |
+
start[0, 0] = end[0, 0] = 0.0
|
130 |
+
|
131 |
+
starts, ends, scores = decode_spans(start, end, top_k, max_answer_len, undesired_tokens)
|
132 |
+
return starts, ends, scores, min_null_score
|
special_tokens_map.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "roberta-base"}
|
vocab.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|