File size: 2,822 Bytes
814ee6b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
"""



## usage

- grok

## 风险评估

- 会干扰 sentencepiece.SentencePieceProcessor的正常使用吗?

"""
import sentencepiece



@property
def vocab_size(self):
    """Returns vocab size"""
    return self.get_piece_size()


def get_vocab(self):
    """Returns vocab as a dict"""
    vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
    # vocab.update(self.added_tokens_encoder)
    return vocab


def _tokenize(self, text):
    """Returns a tokenized string."""
    return self.encode(text, out_type=str)

def _convert_token_to_id(self, token):
    """Converts a token (str) in an id using the vocab."""
    return self.piece_to_id(token)

def _convert_id_to_token(self, index):
    """Converts an index (integer) in a token (str) using the vocab."""
    token = self.IdToPiece(index)
    return token

def convert_ids_to_tokens(self, ids, skip_special_tokens=False):
    """ copy from transformers.PreTrainedTokenizer
    Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and
    added tokens.

    Args:
        ids (`int` or `List[int]`):
            The token id (or token ids) to convert to tokens.
        skip_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not to remove special tokens in the decoding.

    Returns:
        `str` or `List[str]`: The decoded token(s).
    """
    self._added_tokens_decoder = {}  # add by xs
    if isinstance(ids, int):
        if ids in self._added_tokens_decoder:
            return self._added_tokens_decoder[ids].content
        else:
            return self._convert_id_to_token(ids)
    tokens = []
    for index in ids:
        index = int(index)
        if skip_special_tokens and index in self.all_special_ids:
            continue
        if index in self._added_tokens_decoder:
            tokens.append(self._added_tokens_decoder[index].content)
        else:
            tokens.append(self._convert_id_to_token(index))
    return tokens


def encode(self, *args, **kwargs):
    """
    add_special_token 是为了兼容 hf_tokenizer
    """
    kwargs.pop("add_special_tokens", None)
    kwargs.pop("allowed_special", None)
    return self.Encode(*args, **kwargs)


def decode(self, *args, **kwargs):
    kwargs.pop("skip_special_tokens", None)
    return self.Decode(*args, **kwargs)


sentencepiece.SentencePieceProcessor.vocab_size = vocab_size
sentencepiece.SentencePieceProcessor.get_vocab = get_vocab
sentencepiece.SentencePieceProcessor._convert_id_to_token = _convert_id_to_token
sentencepiece.SentencePieceProcessor.convert_ids_to_tokens = convert_ids_to_tokens
# sentencepiece.SentencePieceProcessor.tokenize = _tokenize
sentencepiece.SentencePieceProcessor.encode = encode
sentencepiece.SentencePieceProcessor.decode = decode