zxdu20 commited on
Commit
72985e8
1 Parent(s): 551a50e

Drop icetk dependency

Browse files
Files changed (2) hide show
  1. ice_text.model +2 -2
  2. tokenization_chatglm.py +48 -60
ice_text.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:99871e0c85db81ad7af1028854fd091cd5778c8414ae9d94bbbc10d02c831c21
3
- size 2699926
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e974d9a69c242ce014c88c2b26089270f6198f3c0b700a887666cd3e816f17e
3
+ size 2706249
tokenization_chatglm.py CHANGED
@@ -3,11 +3,10 @@ from typing import List, Optional, Union
3
  import os
4
 
5
  from transformers.tokenization_utils import PreTrainedTokenizer
6
- from icetk.text_tokenizer import TextTokenizer
7
- import icetk.sentencepiece_model_pb2 as sp_model
8
  from transformers.utils import logging, PaddingStrategy
9
  from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
10
  from typing import Dict
 
11
  import numpy as np
12
 
13
  logger = logging.get_logger(__name__)
@@ -17,61 +16,50 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
17
  }
18
 
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  class SPTokenizer:
21
  def __init__(
22
- self,
23
- vocab_file,
24
- max_blank_length=80,
25
- byte_fallback=True,
26
  ):
27
  assert vocab_file is not None
28
  self.vocab_file = vocab_file
29
  self.special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "<unused_0>", "<sop>", "<eop>", "<ENC>", "<dBLOCK>"]
30
  self.max_blank_length = max_blank_length
31
  self.byte_fallback = byte_fallback
32
- self.text_tokenizer = self._build_text_tokenizer(encode_special_tokens=False)
33
- self.special_text_tokenizer = self._build_text_tokenizer(encode_special_tokens=True)
34
 
35
- @staticmethod
36
- def _configure_tokenizer(
37
- text_tokenizer: TextTokenizer,
38
- special_tokens: List[str],
39
- max_blank_length: int,
40
- byte_fallback: bool,
41
- encode_special_tokens=False,
42
- ):
43
- # special token
44
- special_token_type = 4 if encode_special_tokens else 3 # 3 - CONTROL, 4 - USER_DEFINE
45
- for token in special_tokens:
46
- text_tokenizer.proto.pieces.append(
47
- sp_model.ModelProto.SentencePiece(piece=token, score=0.0, type=special_token_type)
48
- )
49
- # whitespaces
50
- for token in [SPTokenizer.get_tab_token()] + [
51
- SPTokenizer.get_blank_token(i) for i in range(2, max_blank_length + 1)
52
- ]:
53
- text_tokenizer.proto.pieces.append(sp_model.ModelProto.SentencePiece(piece=token, score=0.0, type=4))
54
- # byte fallback
55
- if byte_fallback:
56
- text_tokenizer.proto.trainer_spec.byte_fallback = True
57
- for i in range(256):
58
- text_tokenizer.proto.pieces.append(
59
- sp_model.ModelProto.SentencePiece(piece="<0x{:02X}>".format(i), score=0.0, type=6)
60
- )
61
- text_tokenizer.refresh()
62
-
63
- def _build_text_tokenizer(self, encode_special_tokens=False):
64
- tokenizer = TextTokenizer(self.vocab_file)
65
- self._configure_tokenizer(
66
- tokenizer, self.special_tokens, self.max_blank_length, self.byte_fallback, encode_special_tokens
67
- )
68
- return tokenizer
69
-
70
- def _get_text_tokenizer(self, encode_special_tokens=False):
71
- if encode_special_tokens:
72
- return self.special_text_tokenizer
73
- else:
74
- return self.text_tokenizer
75
 
76
  @staticmethod
77
  def get_blank_token(length: int):
@@ -109,7 +97,7 @@ class SPTokenizer:
109
  return text
110
 
111
  def encode(
112
- self, text: str, linebreak=True, whitespaces=True, special_tokens=False, add_dummy_prefix=True
113
  ) -> List[int]:
114
  """
115
  @param text: Text to encode.
@@ -121,14 +109,14 @@ class SPTokenizer:
121
  text = self._preprocess(text, linebreak, whitespaces)
122
  if not add_dummy_prefix:
123
  text = "<n>" + text
124
- tmp = self._get_text_tokenizer(encode_special_tokens=special_tokens).encode(text)
125
  tokens = [x + self.num_image_tokens for x in tmp]
126
  return tokens if add_dummy_prefix else tokens[2:]
127
 
128
- def decode(self, text_ids: List[int], special_tokens=False) -> str:
129
  ids = [int(_id) - self.num_image_tokens for _id in text_ids]
130
  ids = [_id for _id in ids if _id >= 0]
131
- text = self._get_text_tokenizer(encode_special_tokens=special_tokens).decode(ids)
132
  text = text.replace("<n>", "\n")
133
  text = text.replace(SPTokenizer.get_tab_token(), "\t")
134
  for i in range(2, self.max_blank_length + 1):
@@ -136,7 +124,7 @@ class SPTokenizer:
136
  return text
137
 
138
  def tokenize(
139
- self, text: str, linebreak=True, whitespaces=True, special_tokens=False, add_dummy_prefix=True
140
  ) -> List[str]:
141
  """
142
  @param text: Text to encode.
@@ -148,7 +136,7 @@ class SPTokenizer:
148
  text = self._preprocess(text, linebreak, whitespaces)
149
  if not add_dummy_prefix:
150
  text = "<n>" + text
151
- tokens = self._get_text_tokenizer(encode_special_tokens=special_tokens).tokenize(text)
152
  return tokens if add_dummy_prefix else tokens[2:]
153
 
154
  def __getitem__(self, x: Union[int, str]):
@@ -342,12 +330,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
342
  return token_ids_0
343
 
344
  def _pad(
345
- self,
346
- encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
347
- max_length: Optional[int] = None,
348
- padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
349
- pad_to_multiple_of: Optional[int] = None,
350
- return_attention_mask: Optional[bool] = None,
351
  ) -> dict:
352
  """
353
  Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
 
3
  import os
4
 
5
  from transformers.tokenization_utils import PreTrainedTokenizer
 
 
6
  from transformers.utils import logging, PaddingStrategy
7
  from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
8
  from typing import Dict
9
+ import sentencepiece as spm
10
  import numpy as np
11
 
12
  logger = logging.get_logger(__name__)
 
16
  }
17
 
18
 
19
+ class TextTokenizer:
20
+ def __init__(self, model_path):
21
+ self.sp = spm.SentencePieceProcessor()
22
+ self.sp.Load(model_path)
23
+ self.num_tokens = self.sp.vocab_size()
24
+
25
+ def encode(self, text):
26
+ return self.sp.EncodeAsIds(text)
27
+
28
+ def decode(self, ids: List[int]):
29
+ return self.sp.DecodeIds(ids)
30
+
31
+ def tokenize(self, text):
32
+ return self.sp.EncodeAsPieces(text)
33
+
34
+ def convert_tokens_to_ids(self, tokens):
35
+ return [self.sp.PieceToId(token) for token in tokens]
36
+
37
+ def convert_token_to_id(self, token):
38
+ return self.sp.PieceToId(token)
39
+
40
+ def convert_id_to_token(self, idx):
41
+ return self.sp.IdToPiece(idx)
42
+
43
+ def __len__(self):
44
+ return self.num_tokens
45
+
46
+
47
  class SPTokenizer:
48
  def __init__(
49
+ self,
50
+ vocab_file,
51
+ max_blank_length=80,
52
+ byte_fallback=True,
53
  ):
54
  assert vocab_file is not None
55
  self.vocab_file = vocab_file
56
  self.special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "<unused_0>", "<sop>", "<eop>", "<ENC>", "<dBLOCK>"]
57
  self.max_blank_length = max_blank_length
58
  self.byte_fallback = byte_fallback
59
+ self.text_tokenizer = TextTokenizer(vocab_file)
 
60
 
61
+ def _get_text_tokenizer(self):
62
+ return self.text_tokenizer
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
 
64
  @staticmethod
65
  def get_blank_token(length: int):
 
97
  return text
98
 
99
  def encode(
100
+ self, text: str, linebreak=True, whitespaces=True, add_dummy_prefix=True
101
  ) -> List[int]:
102
  """
103
  @param text: Text to encode.
 
109
  text = self._preprocess(text, linebreak, whitespaces)
110
  if not add_dummy_prefix:
111
  text = "<n>" + text
112
+ tmp = self._get_text_tokenizer().encode(text)
113
  tokens = [x + self.num_image_tokens for x in tmp]
114
  return tokens if add_dummy_prefix else tokens[2:]
115
 
116
+ def decode(self, text_ids: List[int]) -> str:
117
  ids = [int(_id) - self.num_image_tokens for _id in text_ids]
118
  ids = [_id for _id in ids if _id >= 0]
119
+ text = self._get_text_tokenizer().decode(ids)
120
  text = text.replace("<n>", "\n")
121
  text = text.replace(SPTokenizer.get_tab_token(), "\t")
122
  for i in range(2, self.max_blank_length + 1):
 
124
  return text
125
 
126
  def tokenize(
127
+ self, text: str, linebreak=True, whitespaces=True, add_dummy_prefix=True
128
  ) -> List[str]:
129
  """
130
  @param text: Text to encode.
 
136
  text = self._preprocess(text, linebreak, whitespaces)
137
  if not add_dummy_prefix:
138
  text = "<n>" + text
139
+ tokens = self._get_text_tokenizer().tokenize(text)
140
  return tokens if add_dummy_prefix else tokens[2:]
141
 
142
  def __getitem__(self, x: Union[int, str]):
 
330
  return token_ids_0
331
 
332
  def _pad(
333
+ self,
334
+ encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
335
+ max_length: Optional[int] = None,
336
+ padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
337
+ pad_to_multiple_of: Optional[int] = None,
338
+ return_attention_mask: Optional[bool] = None,
339
  ) -> dict:
340
  """
341
  Pad encoded inputs (on left/right and up to predefined length or max length in the batch)