Repository: PaddlePaddle/PaddleNLP

Issue:
offset_mapping 偏移错误
# paddlenlp代码
```python
from paddlenlp.transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")

examples = [{'id': '56bf41013aeaaa14008c959c', 'title': 'Super_Bowl_50', 'context': 'This was the first Super Bowl to feature a quarterback on both teams who was the #1 pick in their draft classes. Manning was the #1 selection of the 1998 NFL draft, while Newton was picked first in 2011. The matchup also pits the top two picks of the 2011 draft against each other: Newton for Carolina and Von Miller for Denver. Manning and Newton also set the record for the largest age difference between opposing Super Bowl quarterbacks at 13 years and 48 days (Manning was 39, Newton was 26).', 'question': 'In 2011, who was the first player to be chosen in the NFL draft?', 'answers': ['Newton', 'Newton', 'Newton'], 'answer_starts': [171, 171, 171], 'is_impossible': False}]
def prepare_validation_features(examples, tokenizer):
    contexts = [examples[i]['context'] for i in range(len(examples))]
    questions = [examples[i]['question'] for i in range(len(examples))]

    tokenized_examples = tokenizer(
        questions,
        contexts,
        stride=128,
        max_seq_len=512)

    # For validation, there is no need to compute start and end positions
    for i, tokenized_example in enumerate(tokenized_examples):
        # Grab the sequence corresponding to that example (to know what is the context and what is the question).
        sequence_ids = tokenized_example['token_type_ids']

        # One example can give several spans, this is the index of the example containing this span of text.
        sample_index = tokenized_example['overflow_to_sample']
        tokenized_examples[i]["example_id"] = examples[sample_index]['id']

        # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token
        # position is part of the context or not.
        tokenized_examples[i]["offset_mapping"] = [
            (o if sequence_ids[k] == 1 else None)
            for k, o in enumerate(tokenized_example["offset_mapping"])
        ]

    return tokenized_examples[0]
outputs = prepare_validation_features(examples,tokenizer)
for ids,offsetmap in zip(outputs["input_ids"],outputs["offset_mapping"]):
    if offsetmap:
        print(offsetmap,examples[0]["context"][offsetmap[0]:offsetmap[1]])
        print("="*20)
```
# 结果
```python
(0, 4) This
====================
(5, 8) was
====================
(9, 12) the
====================
(13, 18) first
====================
(19, 24) Super
====================
(25, 29) Bowl
====================
(30, 32) to
====================
(33, 40) feature
====================
(41, 42) a
====================
(43, 54) quarterback
====================
(55, 57) on
====================
(58, 62) both
====================
(63, 68) teams
====================
(69, 72) who
====================
(73, 76) was
====================
(77, 80) the
====================
(81, 81) 
====================
(81, 82) #
====================
(83, 87)  pic
====================
(88, 90)  i
====================
(91, 96)  thei
====================
(97, 102)  draf
====================
(103, 110)  classe
====================
(110, 111) s
```
# huggingface 代码
```python
from transformers import BertTokenizerFast
hg = BertTokenizerFast.from_pretrained("bert-base-uncased")
data = {'id': '56bf41013aeaaa14008c959c', 'title': 'Super_Bowl_50', 'context': 'This was the first Super Bowl to feature a quarterback on both teams who was the #1 pick in their draft classes. Manning was the #1 selection of the 1998 NFL draft, while Newton was picked first in 2011. The matchup also pits the top two picks of the 2011 draft against each other: Newton for Carolina and Von Miller for Denver. Manning and Newton also set the record for the largest age difference between opposing Super Bowl quarterbacks at 13 years and 48 days (Manning was 39, Newton was 26).', 'question': 'In 2011, who was the first player to be chosen in the NFL draft?', 'answers': ['Newton', 'Newton', 'Newton'], 'answer_starts': [171, 171, 171], 'is_impossible': False}
outputs = hg(data["question"],data["context"],return_offsets_mapping=True,return_attention_mask=False,return_token_type_ids=True)
for a,offsetmap,token_type_id in zip(outputs["input_ids"],outputs["offset_mapping"],outputs["token_type_ids"]):
    if token_type_id==1 and offsetmap!=(0,0):
        print(offsetmap,data["context"][offsetmap[0]:offsetmap[1]])
        print("="*20)
```
# 结果正确
```python
(0, 4) This
====================
(5, 8) was
====================
(9, 12) the
====================
(13, 18) first
====================
(19, 24) Super
====================
(25, 29) Bowl
====================
(30, 32) to
====================
(33, 40) feature
====================
(41, 42) a
====================
(43, 54) quarterback
====================
(55, 57) on
====================
(58, 62) both
====================
(63, 68) teams
====================
(69, 72) who
====================
(73, 76) was
====================
(77, 80) the
====================
(81, 82) #
====================
(82, 83) 1
====================
(84, 88) pick
====================
(89, 91) in
====================
(92, 97) their
====================
```


Discussion:
发现是井号`#`造成的

Resolving patch:
diff --git a/paddlenlp/transformers/tokenizer_utils.py b/paddlenlp/transformers/tokenizer_utils.py
--- a/paddlenlp/transformers/tokenizer_utils.py
+++ b/paddlenlp/transformers/tokenizer_utils.py
@@ -1239,15 +1239,15 @@ def get_input_ids(text):
                         if i == len(sub_tokens) - 1:
                             token_offset_mapping.append(
                                 (token_start_offset, token_start_offset +
-                                 len(sub_tokens[i].strip("##"))))
+                                 len(sub_tokens[i].replace("##", ""))))
                             token_start_offset += (
-                                len(sub_tokens[i].strip("##")) + 1)
+                                len(sub_tokens[i].replace("##", "")) + 1)
                         else:
                             token_offset_mapping.append(
                                 (token_start_offset, token_start_offset +
-                                 len(sub_tokens[i].strip("##"))))
+                                 len(sub_tokens[i].replace("##", ""))))
                             token_start_offset += (
-                                len(sub_tokens[i].strip("##")))
+                                len(sub_tokens[i].replace("##", "")))
 
                 token_start_offset = 0
                 for token in token_pair:
@@ -1261,15 +1261,15 @@ def get_input_ids(text):
                         if i == len(sub_tokens) - 1:
                             token_pair_offset_mapping.append(
                                 (token_start_offset, token_start_offset +
-                                 len(sub_tokens[i].strip("##"))))
+                                 len(sub_tokens[i].replace("##", ""))))
                             token_start_offset += (
-                                len(sub_tokens[i].strip("##")) + 1)
+                                len(sub_tokens[i].replace("##", "")) + 1)
                         else:
                             token_pair_offset_mapping.append(
                                 (token_start_offset, token_start_offset +
-                                 len(sub_tokens[i].strip("##"))))
+                                 len(sub_tokens[i].replace("##", ""))))
                             token_start_offset += (
-                                len(sub_tokens[i].strip("##")))
+                                len(sub_tokens[i].replace("##", "")))
 
                 offset = 0
                 while offset < len(second_ids):


Test patch: