Spaces:
Sleeping
Sleeping
Upload app.py with huggingface_hub
Browse files
app.py
ADDED
@@ -0,0 +1,382 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#from conf import *
|
2 |
+
#main_path = "/Volumes/TOSHIBA EXT/temp/kbqa_portable_prj"
|
3 |
+
#main_path = "/Users/svjack/temp/HP_kbqa"
|
4 |
+
#from conf import *
|
5 |
+
#main_path = model_path
|
6 |
+
|
7 |
+
import logging
|
8 |
+
import os
|
9 |
+
import sys
|
10 |
+
from dataclasses import dataclass, field
|
11 |
+
from typing import Optional
|
12 |
+
|
13 |
+
import numpy as np
|
14 |
+
from datasets import ClassLabel, load_dataset, load_metric
|
15 |
+
|
16 |
+
import transformers
|
17 |
+
import transformers.adapters.composition as ac
|
18 |
+
from transformers import (
|
19 |
+
AdapterConfig,
|
20 |
+
AutoConfig,
|
21 |
+
AutoModelForTokenClassification,
|
22 |
+
AutoTokenizer,
|
23 |
+
DataCollatorForTokenClassification,
|
24 |
+
HfArgumentParser,
|
25 |
+
MultiLingAdapterArguments,
|
26 |
+
PreTrainedTokenizerFast,
|
27 |
+
Trainer,
|
28 |
+
TrainingArguments,
|
29 |
+
set_seed,
|
30 |
+
)
|
31 |
+
from transformers.trainer_utils import get_last_checkpoint
|
32 |
+
from transformers.utils import check_min_version
|
33 |
+
from transformers.utils.versions import require_version
|
34 |
+
|
35 |
+
|
36 |
+
import pandas as pd
|
37 |
+
import pickle as pkl
|
38 |
+
from copy import deepcopy
|
39 |
+
import torch
|
40 |
+
from scipy.special import softmax
|
41 |
+
from functools import partial, reduce
|
42 |
+
import json
|
43 |
+
from io import StringIO
|
44 |
+
import re
|
45 |
+
|
46 |
+
from transformers import list_adapters, AutoModelWithHeads
|
47 |
+
|
48 |
+
from collections import defaultdict
|
49 |
+
|
50 |
+
@dataclass
|
51 |
+
class ModelArguments:
|
52 |
+
"""
|
53 |
+
Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
|
54 |
+
"""
|
55 |
+
|
56 |
+
model_name_or_path: str = field(
|
57 |
+
metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
|
58 |
+
)
|
59 |
+
config_name: Optional[str] = field(
|
60 |
+
default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
|
61 |
+
)
|
62 |
+
tokenizer_name: Optional[str] = field(
|
63 |
+
default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
|
64 |
+
)
|
65 |
+
cache_dir: Optional[str] = field(
|
66 |
+
default=None,
|
67 |
+
metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
|
68 |
+
)
|
69 |
+
model_revision: str = field(
|
70 |
+
default="main",
|
71 |
+
metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
|
72 |
+
)
|
73 |
+
use_auth_token: bool = field(
|
74 |
+
default=False,
|
75 |
+
metadata={
|
76 |
+
"help": "Will use the token generated when running `transformers-cli login` (necessary to use this script "
|
77 |
+
"with private models)."
|
78 |
+
},
|
79 |
+
)
|
80 |
+
|
81 |
+
|
82 |
+
@dataclass
|
83 |
+
class DataTrainingArguments:
|
84 |
+
"""
|
85 |
+
Arguments pertaining to what data we are going to input our model for training and eval.
|
86 |
+
"""
|
87 |
+
|
88 |
+
task_name: Optional[str] = field(default="ner", metadata={"help": "The name of the task (ner, pos...)."})
|
89 |
+
dataset_name: Optional[str] = field(
|
90 |
+
default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
|
91 |
+
)
|
92 |
+
dataset_config_name: Optional[str] = field(
|
93 |
+
default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
|
94 |
+
)
|
95 |
+
train_file: Optional[str] = field(
|
96 |
+
default=None, metadata={"help": "The input training data file (a csv or JSON file)."}
|
97 |
+
)
|
98 |
+
validation_file: Optional[str] = field(
|
99 |
+
default=None,
|
100 |
+
metadata={"help": "An optional input evaluation data file to evaluate on (a csv or JSON file)."},
|
101 |
+
)
|
102 |
+
test_file: Optional[str] = field(
|
103 |
+
default=None,
|
104 |
+
metadata={"help": "An optional input test data file to predict on (a csv or JSON file)."},
|
105 |
+
)
|
106 |
+
text_column_name: Optional[str] = field(
|
107 |
+
default=None, metadata={"help": "The column name of text to input in the file (a csv or JSON file)."}
|
108 |
+
)
|
109 |
+
label_column_name: Optional[str] = field(
|
110 |
+
default=None, metadata={"help": "The column name of label to input in the file (a csv or JSON file)."}
|
111 |
+
)
|
112 |
+
overwrite_cache: bool = field(
|
113 |
+
default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
|
114 |
+
)
|
115 |
+
preprocessing_num_workers: Optional[int] = field(
|
116 |
+
default=None,
|
117 |
+
metadata={"help": "The number of processes to use for the preprocessing."},
|
118 |
+
)
|
119 |
+
pad_to_max_length: bool = field(
|
120 |
+
default=False,
|
121 |
+
metadata={
|
122 |
+
"help": "Whether to pad all samples to model maximum sentence length. "
|
123 |
+
"If False, will pad the samples dynamically when batching to the maximum length in the batch. More "
|
124 |
+
"efficient on GPU but very bad for TPU."
|
125 |
+
},
|
126 |
+
)
|
127 |
+
max_train_samples: Optional[int] = field(
|
128 |
+
default=None,
|
129 |
+
metadata={
|
130 |
+
"help": "For debugging purposes or quicker training, truncate the number of training examples to this "
|
131 |
+
"value if set."
|
132 |
+
},
|
133 |
+
)
|
134 |
+
max_eval_samples: Optional[int] = field(
|
135 |
+
default=None,
|
136 |
+
metadata={
|
137 |
+
"help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
|
138 |
+
"value if set."
|
139 |
+
},
|
140 |
+
)
|
141 |
+
max_predict_samples: Optional[int] = field(
|
142 |
+
default=None,
|
143 |
+
metadata={
|
144 |
+
"help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
|
145 |
+
"value if set."
|
146 |
+
},
|
147 |
+
)
|
148 |
+
label_all_tokens: bool = field(
|
149 |
+
default=False,
|
150 |
+
metadata={
|
151 |
+
"help": "Whether to put the label for one word on all tokens of generated by that word or just on the "
|
152 |
+
"one (in which case the other tokens will have a padding index)."
|
153 |
+
},
|
154 |
+
)
|
155 |
+
return_entity_level_metrics: bool = field(
|
156 |
+
default=False,
|
157 |
+
metadata={"help": "Whether to return all the entity levels during evaluation or just the overall ones."},
|
158 |
+
)
|
159 |
+
|
160 |
+
def __post_init__(self):
|
161 |
+
if self.dataset_name is None and self.train_file is None and self.validation_file is None:
|
162 |
+
raise ValueError("Need either a dataset name or a training/validation file.")
|
163 |
+
else:
|
164 |
+
if self.train_file is not None:
|
165 |
+
extension = self.train_file.split(".")[-1]
|
166 |
+
assert extension in ["csv", "json"], "`train_file` should be a csv or a json file."
|
167 |
+
if self.validation_file is not None:
|
168 |
+
extension = self.validation_file.split(".")[-1]
|
169 |
+
assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."
|
170 |
+
self.task_name = self.task_name.lower()
|
171 |
+
|
172 |
+
import os
|
173 |
+
|
174 |
+
#p0 = os.path.join(main_path, "sel_ner/ner_data_args.pkl")
|
175 |
+
p0 = "sel_ner/ner_data_args.pkl"
|
176 |
+
assert os.path.exists(p0)
|
177 |
+
with open(p0, "rb") as f:
|
178 |
+
t4 = pkl.load(f)
|
179 |
+
|
180 |
+
model_args, data_args, training_args, adapter_args = map(deepcopy, t4)
|
181 |
+
|
182 |
+
zh_model = AutoModelWithHeads.from_pretrained("bert-base-chinese")
|
183 |
+
|
184 |
+
#config_path = "/Users/svjack/temp/ner_trans/adapter_ner_data/test-sel-ner/sel_ner/adapter_config.json"
|
185 |
+
#adapter_path = "/Users/svjack/temp/ner_trans/adapter_ner_data/test-sel-ner/sel_ner"
|
186 |
+
config_path = "sel_ner/adapter_config.json"
|
187 |
+
adapter_path = "sel_ner"
|
188 |
+
#config_path = os.path.join(main_path ,"sel_ner/adapter_config.json")
|
189 |
+
#adapter_path = os.path.join(main_path ,"sel_ner")
|
190 |
+
|
191 |
+
config = AdapterConfig.load(config_path)
|
192 |
+
zh_model.load_adapter(adapter_path, config=config)
|
193 |
+
zh_model.set_active_adapters(['sel_ner'])
|
194 |
+
|
195 |
+
def single_sent_pred(input_text, tokenizer, model):
|
196 |
+
input_ = tokenizer(input_text)
|
197 |
+
input_ids = input_["input_ids"]
|
198 |
+
output = model(torch.Tensor([input_ids]).type(torch.LongTensor))
|
199 |
+
output_prob = softmax(output.logits.detach().numpy()[0], axis = -1)
|
200 |
+
token_list = tokenizer.convert_ids_to_tokens(input_ids)
|
201 |
+
assert len(token_list) == len(output_prob)
|
202 |
+
return token_list, output_prob
|
203 |
+
|
204 |
+
def single_pred_to_df(token_list, output_prob, label_list):
|
205 |
+
assert output_prob.shape[0] == len(token_list) and output_prob.shape[1] == len(label_list)
|
206 |
+
pred_label_list = pd.Series(output_prob.argmax(axis = -1)).map(
|
207 |
+
lambda idx: label_list[idx]
|
208 |
+
).tolist()
|
209 |
+
return pd.concat(list(map(pd.Series, [token_list, pred_label_list])), axis = 1)
|
210 |
+
|
211 |
+
def token_l_to_nest_l(token_l, prefix = "##"):
|
212 |
+
req = []
|
213 |
+
#req.append([])
|
214 |
+
#### token_l must startswith [CLS]
|
215 |
+
assert token_l[0] == "[CLS]"
|
216 |
+
for ele in token_l:
|
217 |
+
if not ele.startswith(prefix):
|
218 |
+
req.append([ele])
|
219 |
+
else:
|
220 |
+
req[-1].append(ele)
|
221 |
+
return req
|
222 |
+
|
223 |
+
def list_window_collect(l, w_size = 1, drop_NONE = False):
|
224 |
+
assert len(l) >= w_size
|
225 |
+
req = []
|
226 |
+
for i in range(len(l)):
|
227 |
+
l_slice = l[i: i + w_size]
|
228 |
+
l_slice += [None] * (w_size - len(l_slice))
|
229 |
+
req.append(l_slice)
|
230 |
+
if drop_NONE:
|
231 |
+
return list(filter(lambda x: None not in x, req))
|
232 |
+
return req
|
233 |
+
|
234 |
+
def same_pkt_l(l0, l1):
|
235 |
+
l0_size_l = list(map(len, l0))
|
236 |
+
assert sum(l0_size_l) == len(l1)
|
237 |
+
cum_l0_size = np.cumsum(l0_size_l).tolist()
|
238 |
+
slice_l = list_window_collect(cum_l0_size, 2, drop_NONE=True)
|
239 |
+
slice_l = [[0 ,slice_l[0][0]]] + slice_l
|
240 |
+
slice_df = pd.DataFrame(slice_l)
|
241 |
+
return (l0, slice_df.apply(lambda s: l1[s[0]:s[1]], axis = 1).tolist())
|
242 |
+
|
243 |
+
|
244 |
+
def cnt_backtrans_slice(token_list, label_list, prefix = "##",
|
245 |
+
token_agg_func = lambda x: x[0] if len(x) == 1 else "".join([x[0]] + list(map(lambda y: y[len("##"):], x[1:]))),
|
246 |
+
label_agg_func = lambda x: x[0] if len(x) == 1 else pd.Series(x).value_counts().index.tolist()[0]
|
247 |
+
):
|
248 |
+
token_nest_list = token_l_to_nest_l(token_list, prefix=prefix)
|
249 |
+
token_nest_list, label_nest_list = same_pkt_l(token_nest_list, label_list)
|
250 |
+
token_list_req = list(map(token_agg_func, token_nest_list))
|
251 |
+
label_list_req = list(map(label_agg_func, label_nest_list))
|
252 |
+
return (token_list_req, label_list_req)
|
253 |
+
|
254 |
+
def from_text_to_final(input_text, tokenizer, model, label_list):
|
255 |
+
token_list, output_prob = single_sent_pred(input_text, tokenizer, model)
|
256 |
+
token_pred_df = single_pred_to_df(token_list, output_prob, label_list)
|
257 |
+
token_list_, label_list_ = token_pred_df[0].tolist(), token_pred_df[1].tolist()
|
258 |
+
token_pred_df_reduce = pd.DataFrame(list(zip(*cnt_backtrans_slice(token_list_, label_list_))))
|
259 |
+
return token_pred_df_reduce
|
260 |
+
|
261 |
+
|
262 |
+
tokenizer_name_or_path = model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path
|
263 |
+
|
264 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
265 |
+
tokenizer_name_or_path,
|
266 |
+
cache_dir=model_args.cache_dir,
|
267 |
+
use_fast=True,
|
268 |
+
revision=model_args.model_revision,
|
269 |
+
use_auth_token=True if model_args.use_auth_token else None,
|
270 |
+
)
|
271 |
+
|
272 |
+
label_list = ['O-TAG', 'E-TAG', 'T-TAG']
|
273 |
+
|
274 |
+
### fix eng with " "
|
275 |
+
### used when ner_model input with some eng-string fillwith " "
|
276 |
+
def fill_str(sent ,str_):
|
277 |
+
is_en = False
|
278 |
+
if re.findall("[a-zA-Z0-9 ]+", str_) and re.findall("[a-zA-Z0-9 ]+", str_)[0] == str_:
|
279 |
+
is_en = True
|
280 |
+
if not is_en:
|
281 |
+
return str_
|
282 |
+
find_part = re.findall("([{} ]+)".format(str_), text)
|
283 |
+
assert find_part
|
284 |
+
find_part = sorted(filter(lambda x: x.replace(" ", "") == str_.replace(" ", "") ,find_part), key = len, reverse = True)[0]
|
285 |
+
assert find_part in sent
|
286 |
+
return find_part
|
287 |
+
|
288 |
+
def for_loop_detect(s, invalid_tag = "O-TAG", sp_token = "123454321"):
|
289 |
+
assert type(s) == type(pd.Series())
|
290 |
+
char_list = s.iloc[0]
|
291 |
+
tag_list = s.iloc[1]
|
292 |
+
assert len(char_list) == len(tag_list)
|
293 |
+
req = defaultdict(list)
|
294 |
+
pre_tag = ""
|
295 |
+
for idx, tag in enumerate(tag_list):
|
296 |
+
if tag == invalid_tag or tag != pre_tag:
|
297 |
+
for k in req.keys():
|
298 |
+
if req[k][-1] != invalid_tag:
|
299 |
+
req[k].append(sp_token)
|
300 |
+
if tag != pre_tag and tag != invalid_tag:
|
301 |
+
char = char_list[idx]
|
302 |
+
req[tag].append(char)
|
303 |
+
elif tag != invalid_tag:
|
304 |
+
char = char_list[idx]
|
305 |
+
req[tag].append(char)
|
306 |
+
pre_tag = tag
|
307 |
+
req = dict(map(lambda t2: (
|
308 |
+
t2[0],
|
309 |
+
list(
|
310 |
+
filter(lambda x: x.strip() ,"".join(t2[1]).split(sp_token))
|
311 |
+
)
|
312 |
+
), req.items()))
|
313 |
+
return req
|
314 |
+
|
315 |
+
def ner_entity_type_predict_only(question):
|
316 |
+
assert type(question) == type("")
|
317 |
+
question = question.replace(" ", "")
|
318 |
+
ner_df = from_text_to_final(
|
319 |
+
" ".join(list(question)),
|
320 |
+
tokenizer,
|
321 |
+
zh_model,
|
322 |
+
label_list
|
323 |
+
)
|
324 |
+
assert ner_df.shape[0] == len(question) + 2
|
325 |
+
### [UNK] filling
|
326 |
+
ner_df[0] = ["[CLS]"] + list(question) + ["[SEP]"]
|
327 |
+
et_dict = for_loop_detect(ner_df.T.apply(lambda x: x.tolist(), axis = 1))
|
328 |
+
return et_dict
|
329 |
+
|
330 |
+
import gradio as gr
|
331 |
+
|
332 |
+
example_sample = [
|
333 |
+
"宁波在哪个省份?",
|
334 |
+
"美国的通货是什么?",
|
335 |
+
]
|
336 |
+
|
337 |
+
|
338 |
+
demo = gr.Interface(
|
339 |
+
fn=ner_entity_type_predict_only,
|
340 |
+
inputs="text",
|
341 |
+
outputs="json",
|
342 |
+
title=f"Chinese Question Entity Property decomposition 🌧️ demonstration",
|
343 |
+
examples=example_sample if example_sample else None,
|
344 |
+
cache_examples = False
|
345 |
+
)
|
346 |
+
|
347 |
+
demo.launch(server_name=None, server_port=None)
|
348 |
+
|
349 |
+
'''
|
350 |
+
rep = requests.post(
|
351 |
+
url = "http://localhost:8855/extract_et",
|
352 |
+
data = {
|
353 |
+
"question": "哈利波特的作者是谁?"
|
354 |
+
}
|
355 |
+
)
|
356 |
+
json.loads(rep.content.decode())
|
357 |
+
|
358 |
+
@csrf_exempt
|
359 |
+
def extract_et(request):
|
360 |
+
assert request.method == "POST"
|
361 |
+
post_data = request.POST
|
362 |
+
question = post_data["question"]
|
363 |
+
assert type(question) == type("")
|
364 |
+
#question = "宁波在哪个省?"
|
365 |
+
#abc = do_search(question)
|
366 |
+
et_dict = ner_entity_type_predict_only(question)
|
367 |
+
assert type(et_dict) == type({})
|
368 |
+
return HttpResponse(json.dumps(et_dict))
|
369 |
+
|
370 |
+
if __name__ == "__main__":
|
371 |
+
from_text_to_final("宁波在哪个省?",
|
372 |
+
tokenizer,
|
373 |
+
zh_model,
|
374 |
+
label_list
|
375 |
+
)
|
376 |
+
|
377 |
+
from_text_to_final("美国的通货是什么?",
|
378 |
+
tokenizer,
|
379 |
+
zh_model,
|
380 |
+
label_list
|
381 |
+
)
|
382 |
+
'''
|