Andrazp commited on
Commit
d10f96a
1 Parent(s): ea93c89

adding vocabulary files for alberto tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.py +238 -0
  2. vocab.txt +0 -0
tokenizer.py ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2018 The Google AI Language Team Authors, The HuggingFace Inc. team,
3
+ # and Marco Polignano.
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ """Tokenization classes for Italian AlBERTo models."""
18
+ import collections
19
+ import logging
20
+ import os
21
+ import re
22
+ import logger
23
+
24
+ try:
25
+ from ekphrasis.classes.preprocessor import TextPreProcessor
26
+ from ekphrasis.classes.tokenizer import SocialTokenizer
27
+ from ekphrasis.dicts.emoticons import emoticons
28
+ except ImportError:
29
+ #logger.warning(
30
+ # "You need to install ekphrasis to use AlBERToTokenizer"
31
+ # "pip install ekphrasis"
32
+ #)
33
+ from pip._internal import main as pip
34
+ pip(['install', '--user', 'ekphrasis'])
35
+ from ekphrasis.classes.preprocessor import TextPreProcessor
36
+ from ekphrasis.classes.tokenizer import SocialTokenizer
37
+ from ekphrasis.dicts.emoticons import emoticons
38
+
39
+ try:
40
+ import numpy as np
41
+ except ImportError:
42
+ logger.warning(
43
+ "You need to install numpy to use AlBERToTokenizer"
44
+ "pip install numpy"
45
+ )
46
+ from pip._internal import main as pip
47
+ pip(['install', '--user', 'pandas'])
48
+ import pandas as pd
49
+
50
+ try:
51
+ from transformers import BertTokenizer, WordpieceTokenizer
52
+ from transformers.tokenization_bert import load_vocab
53
+ except ImportError:
54
+ logger.warning(
55
+ "You need to install pytorch-transformers to use AlBERToTokenizer"
56
+ "pip install pytorch-transformers"
57
+ )
58
+ from pip._internal import main as pip
59
+ pip(['install', '--user', 'pytorch-transformers'])
60
+ from transformers import BertTokenizer, WordpieceTokenizer
61
+ from transformers.tokenization_bert import load_vocab
62
+
63
+ text_processor = TextPreProcessor(
64
+ # terms that will be normalized
65
+ normalize=['url', 'email', 'user', 'percent', 'money', 'phone', 'time', 'date', 'number'],
66
+ # terms that will be annotated
67
+ annotate={"hashtag"},
68
+ fix_html=True, # fix HTML tokens
69
+
70
+ unpack_hashtags=True, # perform word segmentation on hashtags
71
+
72
+ # select a tokenizer. You can use SocialTokenizer, or pass your own
73
+ # the tokenizer, should take as input a string and return a list of tokens
74
+ tokenizer=SocialTokenizer(lowercase=True).tokenize,
75
+ dicts=[emoticons]
76
+ )
77
+
78
+ class AlBERTo_Preprocessing(object):
79
+ def __init__(self, do_lower_case=True, **kwargs):
80
+ self.do_lower_case = do_lower_case
81
+
82
+ def preprocess(self, text):
83
+ if self.do_lower_case:
84
+ text = text.lower()
85
+ text = str(" ".join(text_processor.pre_process_doc(text)))
86
+ text = re.sub(r'[^a-zA-ZÀ-ú</>!?♥♡\s\U00010000-\U0010ffff]', ' ', text)
87
+ text = re.sub(r'\s+', ' ', text)
88
+ text = re.sub(r'(\w)\1{2,}', r'\1\1', text)
89
+ text = re.sub(r'^\s', '', text)
90
+ text = re.sub(r'\s$', '', text)
91
+ return text
92
+
93
+ class AlBERToTokenizer(BertTokenizer):
94
+
95
+ def __init__(self, vocab_file, do_lower_case=True,
96
+ do_basic_tokenize=True, do_char_tokenize=False, do_wordpiece_tokenize=False, do_preprocessing = True, unk_token='[UNK]',
97
+ sep_token='[SEP]',
98
+ pad_token='[PAD]', cls_token='[CLS]', mask_token='[MASK]', **kwargs):
99
+ super(BertTokenizer, self).__init__(
100
+ unk_token=unk_token, sep_token=sep_token, pad_token=pad_token,
101
+ cls_token=cls_token, mask_token=mask_token, **kwargs)
102
+
103
+ self.do_wordpiece_tokenize = do_wordpiece_tokenize
104
+ self.do_lower_case = do_lower_case
105
+ self.vocab_file = vocab_file
106
+ self.do_basic_tokenize = do_basic_tokenize
107
+ self.do_char_tokenize = do_char_tokenize
108
+ self.unk_token = unk_token
109
+ self.do_preprocessing = do_preprocessing
110
+
111
+ if not os.path.isfile(vocab_file):
112
+ raise ValueError(
113
+ "Can't find a vocabulary file at path '{}'.".format(vocab_file))
114
+
115
+ self.vocab = load_vocab(vocab_file)
116
+ self.ids_to_tokens = collections.OrderedDict(
117
+ [(ids, tok) for tok, ids in self.vocab.items()])
118
+
119
+ if do_wordpiece_tokenize:
120
+ self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab,
121
+ unk_token=self.unk_token)
122
+
123
+ self.base_bert_tok = BertTokenizer(vocab_file=self.vocab_file, do_lower_case=do_lower_case,
124
+ unk_token=unk_token, sep_token=sep_token, pad_token=pad_token,
125
+ cls_token=cls_token, mask_token=mask_token, **kwargs)
126
+
127
+ def _convert_token_to_id(self, token):
128
+ """Converts a token (str/unicode) to an id using the vocab."""
129
+ # if token[:2] == '##':
130
+ # token = token[2:]
131
+
132
+ return self.vocab.get(token, self.vocab.get(self.unk_token))
133
+
134
+ def convert_token_to_id(self, token):
135
+ return self._convert_token_to_id(token)
136
+
137
+ return self.vocab.get(token, self.vocab.get(self.unk_token))
138
+
139
+ def _convert_id_to_token(self, id):
140
+ # if token[:2] == '##':
141
+ # token = token[2:]
142
+
143
+ return list(self.vocab.keys())[int(id)]
144
+ def convert_id_to_token(self, id):
145
+ return self._convert_id_to_token(id)
146
+
147
+ def _convert_tokens_to_string(self,tokens):
148
+ """Converts a sequence of tokens (string) to a single string."""
149
+ out_string = ' '.join(tokens).replace('##', '').strip()
150
+ return out_string
151
+
152
+ def convert_tokens_to_string(self,tokens):
153
+ return self._convert_tokens_to_string(tokens)
154
+
155
+ def _tokenize(self, text, never_split=None, **kwargs):
156
+ if self.do_preprocessing:
157
+ if self.do_lower_case:
158
+ text = text.lower()
159
+ text = str(" ".join(text_processor.pre_process_doc(text)))
160
+ text = re.sub(r'[^a-zA-ZÀ-ú</>!?♥♡\s\U00010000-\U0010ffff]', ' ', text)
161
+ text = re.sub(r'\s+', ' ', text)
162
+ text = re.sub(r'(\w)\1{2,}', r'\1\1', text)
163
+ text = re.sub(r'^\s', '', text)
164
+ text = re.sub(r'\s$', '', text)
165
+ # print(s)
166
+
167
+ split_tokens = [text]
168
+ if self.do_wordpiece_tokenize:
169
+ wordpiece_tokenizer = WordpieceTokenizer(self.vocab,self.unk_token)
170
+ split_tokens = wordpiece_tokenizer.tokenize(text)
171
+
172
+ elif self.do_char_tokenize:
173
+ tokenizer = CharacterTokenizer(self.vocab, self.unk_token)
174
+ split_tokens = tokenizer.tokenize(text)
175
+
176
+ elif self.do_basic_tokenize:
177
+ """Tokenizes a piece of text."""
178
+ split_tokens = self.base_bert_tok.tokenize(text)
179
+
180
+ return split_tokens
181
+
182
+ def tokenize(self, text, never_split=None, **kwargs):
183
+ return self._tokenize(text, never_split)
184
+
185
+
186
+ class CharacterTokenizer(object):
187
+ """Runs Character tokenziation."""
188
+
189
+ def __init__(self, vocab, unk_token,
190
+ max_input_chars_per_word=100, with_markers=True):
191
+ """Constructs a CharacterTokenizer.
192
+ Args:
193
+ vocab: Vocabulary object.
194
+ unk_token: A special symbol for out-of-vocabulary token.
195
+ with_markers: If True, "#" is appended to each output character except the
196
+ first one.
197
+ """
198
+ self.vocab = vocab
199
+ self.unk_token = unk_token
200
+ self.max_input_chars_per_word = max_input_chars_per_word
201
+ self.with_markers = with_markers
202
+
203
+ def tokenize(self, text):
204
+ """Tokenizes a piece of text into characters.
205
+
206
+ For example:
207
+ input = "apple"
208
+ output = ["a", "##p", "##p", "##l", "##e"] (if self.with_markers is True)
209
+ output = ["a", "p", "p", "l", "e"] (if self.with_markers is False)
210
+ Args:
211
+ text: A single token or whitespace separated tokens.
212
+ This should have already been passed through `BasicTokenizer`.
213
+ Returns:
214
+ A list of characters.
215
+ """
216
+
217
+ output_tokens = []
218
+ for i, char in enumerate(text):
219
+ if char not in self.vocab:
220
+ output_tokens.append(self.unk_token)
221
+ continue
222
+
223
+ if self.with_markers and i != 0:
224
+ output_tokens.append('##' + char)
225
+ else:
226
+ output_tokens.append(char)
227
+
228
+ return output_tokens
229
+
230
+ if __name__== "__main__":
231
+ a = AlBERTo_Preprocessing(do_lower_case=True)
232
+ s = "#IlGOverno presenta le linee guida sulla scuola #labuonascuola - http://t.co/SYS1T9QmQN"
233
+ b = a.preprocess(s)
234
+ print(b)
235
+
236
+ c =AlBERToTokenizer(do_lower_case=True,vocab_file="vocab.txt", do_preprocessing=True)
237
+ d = c.tokenize(s)
238
+ print(d)
vocab.txt ADDED
The diff for this file is too large to render. See raw diff