Lihuchen commited on
Commit
fa0a93c
1 Parent(s): d800039
Files changed (6) hide show
  1. acrobert.py +127 -0
  2. constant.py +0 -0
  3. maddog.py +981 -0
  4. popularity.py +31 -0
  5. stopWords.txt +127 -0
  6. utils.py +209 -0
acrobert.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ from math import exp
3
+ import torch
4
+ from torch import nn
5
+ from transformers import BertTokenizer, BertForNextSentencePrediction
6
+ import utils
7
+ from maddog import Extractor
8
+ import spacy
9
+ import constant
10
+
11
+ nlp = spacy.load("en_core_web_sm")
12
+ ruleExtractor = Extractor()
13
+ kb = utils.load_acronym_kb('acronym_kb.json')
14
+ model_path='acrobert.pt'
15
+
16
+ class AcronymBERT(nn.Module):
17
+ def __init__(self, model_name="bert-base-uncased", device='cpu'):
18
+ super().__init__()
19
+ self.device = device
20
+ self.model = BertForNextSentencePrediction.from_pretrained(model_name)
21
+ self.tokenizer = BertTokenizer.from_pretrained(model_name)
22
+
23
+ def forward(self, sentence):
24
+
25
+ samples = self.tokenizer(sentence, padding=True, return_tensors='pt', truncation=True)["input_ids"]
26
+ samples = samples.to(self.device)
27
+ outputs = self.model(samples).logits
28
+ scores = nn.Softmax(dim=1)(outputs)[:, 0]
29
+
30
+ return scores
31
+
32
+ model = AcronymBERT(device='cpu')
33
+ model.load_state_dict(torch.load(model_path, map_location='cpu'))
34
+
35
+ def softmax(elements):
36
+ total = sum([exp(e) for e in elements])
37
+ return exp(elements[0]) / total
38
+
39
+
40
+ def predict(topk, model, short_form, context, batch_size, acronym_kb, device):
41
+ ori_candidate = utils.get_candidate(acronym_kb, short_form, can_num=10)
42
+ long_terms = [str.lower(can) for can in ori_candidate]
43
+ scores = cal_score(model.model, model.tokenizer, long_terms, context, batch_size, device)
44
+ #indexes = [np.argmax(scores)]
45
+ topk = min(len(scores), topk)
46
+ indexes = np.array(scores).argsort()[::-1][:topk]
47
+ names = [ori_candidate[i] for i in indexes]
48
+ return names
49
+
50
+
51
+ def cal_score(model, tokenizer, long_forms, contexts, batch_size, device):
52
+ ps = list()
53
+ for index in range(0, len(long_forms), batch_size):
54
+ batch_lf = long_forms[index:index + batch_size]
55
+ batch_ctx = [contexts] * len(batch_lf)
56
+ encoding = tokenizer(batch_lf, batch_ctx, return_tensors="pt", padding=True, truncation=True, max_length=400).to(device)
57
+ outputs = model(**encoding)
58
+ logits = outputs.logits.cpu().detach().numpy()
59
+ p = [softmax(lg) for lg in logits]
60
+ ps.extend(p)
61
+ return ps
62
+
63
+
64
+ def dog_extract(sentence):
65
+ tokens = [t.text for t in nlp(sentence) if len(t.text.strip()) > 0]
66
+ rulebased_pairs = ruleExtractor.extract(tokens, constant.RULES)
67
+ return rulebased_pairs
68
+
69
+
70
+ def acrobert(sentence, model, device):
71
+
72
+ model.to(device)
73
+
74
+ #params = sum(p.numel() for p in model.parameters() if p.requires_grad)
75
+ #print(params)
76
+
77
+ tokens = [t.text for t in nlp(sentence) if len(t.text.strip()) > 0]
78
+ rulebased_pairs = ruleExtractor.extract(tokens, constant.RULES)
79
+
80
+ results = list()
81
+ for acronym in rulebased_pairs.keys():
82
+ if rulebased_pairs[acronym][0] != '':
83
+ results.append((acronym, rulebased_pairs[acronym][0]))
84
+ else:
85
+ pred = predict(1, model, acronym, sentence, batch_size=10, acronym_kb=kb, device=device)
86
+ results.append((acronym, pred[0]))
87
+ return results
88
+
89
+
90
+ def popularity(sentence):
91
+
92
+ tokens = [t.text for t in nlp(sentence) if len(t.text.strip()) > 0]
93
+ rulebased_pairs = ruleExtractor.extract(tokens, constant.RULES)
94
+
95
+ results = list()
96
+ for acronym in rulebased_pairs.keys():
97
+ if rulebased_pairs[acronym][0] != '':
98
+ results.append((acronym, rulebased_pairs[acronym][0]))
99
+ else:
100
+
101
+ pred = utils.get_candidate(kb, acronym, can_num=1)
102
+ results.append((acronym, pred[0]))
103
+ return results
104
+
105
+
106
+ def acronym_linker(sentence, mode='acrobert', model=model, device='cpu'):
107
+ if mode == 'acrobert':
108
+ return acrobert(sentence, model, device)
109
+ if mode == 'pop':
110
+ return popularity(sentence)
111
+ raise Exception('mode name should in this list [acrobert, pop]')
112
+
113
+
114
+ if __name__ == '__main__':
115
+ #sentence = \
116
+ #"This new genome assembly and the annotation are tagged as a RefSeq genome by NCBI and thus provide substantially enhanced genomic resources for future research involving S. scovelli."
117
+
118
+ #sentence = """ There have been initiated several projects to modernize the network of ECB
119
+ #corridors, financed from ispa funds and state-guaranteed loans from international
120
+ #financial institutions."""
121
+ # sentence = """A whistleblower like monologist Mike Daisey gets targeted as a scapegoat who must
122
+ # be discredited and diminished in the public ’s eye. More often than not, PR is
123
+ # a preemptive process. Celebrity publicists are paid lots of money to keep certain
124
+ # stories out of the news."""
125
+ sentence = "AI is the ability of a digital computer or computer-controlled robot to perform tasks commonly associated with intelligent beings, including NLP that processes text or document"
126
+ results = acronym_linker(sentence)
127
+ print(results)
constant.py ADDED
The diff for this file is too large to render. See raw diff
 
maddog.py ADDED
@@ -0,0 +1,981 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ '''
2
+ from https://github.com/amirveyseh/MadDog under CC BY-NC-SA 4.0
3
+ '''
4
+
5
+ import string
6
+
7
+ if __name__ != "__main__":
8
+ import spacy
9
+
10
+ nlp = spacy.load("en_core_web_sm")
11
+
12
+ with open('stopWords.txt') as file:
13
+ stop_words = [l.strip() for l in file.readlines()]
14
+
15
+
16
+ class Extractor:
17
+ def __init__(self):
18
+ pass
19
+
20
+ def short_extract(self, sentence, threshold, starting_lower_case, ignore_dot=False):
21
+ shorts = []
22
+ for i, t in enumerate(sentence):
23
+ if ignore_dot:
24
+ t = t.replace('.', '')
25
+ # t = t.replace('-','')
26
+ if len(t) == 0:
27
+ continue
28
+ # FIXED [issue: of an enhanced Node B ( eNB ) ]
29
+ if not starting_lower_case:
30
+ if t[0].isupper() and len([c for c in t if c.isupper()]) / len(t) > threshold and 2 <= len(t) <= 10:
31
+ shorts.append(i)
32
+ else:
33
+ if len([c for c in t if c.isupper()]) / len(t) > threshold and 2 <= len(t) <= 10:
34
+ shorts.append(i)
35
+ return shorts
36
+
37
+ def extract_cand_long(self, sentence, token, ind, ignore_punc=False, add_punc=False, small_window=False):
38
+ '''
39
+ extract candidate long form of the form "long form (short form)" or "short form (long form)"
40
+
41
+ :param sentence: tokenized sentence
42
+ :param token: acronym
43
+ :param ind: position of the acronym
44
+ :return: candidate long form, candidate is on left or right of the short form
45
+ '''
46
+ if not small_window:
47
+ long_cand_length = min([len(token) + 10, len(token) * 3])
48
+ else:
49
+ long_cand_length = min([len(token) + 5, len(token) * 2])
50
+ cand_long = []
51
+ cand_long_index = []
52
+ left = True
53
+ right_ind = 1
54
+ left_ind = 1
55
+ # FIXED [issue: ]
56
+ if add_punc:
57
+ excluded_puncs = ['=', ':']
58
+ else:
59
+ excluded_puncs = []
60
+ # FIXED [issue: such as Latent Semantic Analysis ( LSA ; )]
61
+ if ignore_punc:
62
+ while ind + right_ind < len(sentence) and sentence[ind + right_ind] in [p for p in string.punctuation if
63
+ p != '(' and p != ')' and p not in excluded_puncs]:
64
+ right_ind += 1
65
+ while ind - left_ind > 0 and sentence[ind - left_ind] in [p for p in string.punctuation if
66
+ p != '(' and p != ')' and p not in excluded_puncs]:
67
+ left_ind -= 1
68
+ ####
69
+ if ind < len(sentence) - 2 - right_ind and (
70
+ sentence[ind + right_ind] == '(' or sentence[ind + right_ind] == '=' or sentence[
71
+ ind + right_ind] in excluded_puncs):
72
+ left = False
73
+ for j in range(ind + right_ind + 1, min([ind + right_ind + 1 + long_cand_length, len(sentence)])):
74
+ if sentence[j] != ')':
75
+ cand_long.append(sentence[j])
76
+ cand_long_index.append(j)
77
+ else:
78
+ break
79
+ elif 1 < ind - (left_ind - 1) and ind + right_ind < len(sentence) and (
80
+ (sentence[ind - left_ind] == '(' and sentence[ind + right_ind] == ')') or sentence[
81
+ ind - left_ind] in excluded_puncs):
82
+ for k in range(0, long_cand_length):
83
+ j = ind - left_ind - 1 - k
84
+ if j > -1:
85
+ cand_long.insert(0, sentence[j])
86
+ cand_long_index.insert(0, j)
87
+ return cand_long, cand_long_index, left
88
+
89
+ # FIXED [issue: The Stopping Trained in America PhDs from Leaving the Economy Act ( or STAPLE Act ) has bee introduced]
90
+ def extract_high_recall_cand_long(self, sentence, token, ind, small_window=False, left=False):
91
+ '''
92
+ Find the candidate long form for a give acronym for high recall extraction
93
+ example: The Stopping Trained in America PhDs from Leaving the Economy Act ( or STAPLE Act ) has bee introduced
94
+
95
+ :param sentence:
96
+ :param token:
97
+ :param ind:
98
+ :param small_window:
99
+ :return:
100
+ '''
101
+ long_cand_length = min([len(token) + 10, len(token) * 3])
102
+ cand_long = []
103
+ cand_long_index = []
104
+ if not left:
105
+ for j in range(ind + 1, min([ind + long_cand_length, len(sentence)])):
106
+ cand_long.append(sentence[j])
107
+ cand_long_index.append(j)
108
+ else:
109
+ for k in range(0, long_cand_length):
110
+ j = ind - 1 - k
111
+ if j > -1:
112
+ cand_long.insert(0, sentence[j])
113
+ cand_long_index.insert(0, j)
114
+ return cand_long, cand_long_index, left
115
+
116
+ def create_diction(self, sentence, labels, all_acronyms=True, tag='', map_chars=False, diction={}):
117
+ '''
118
+ convert sequential labels into {short-form: long-form} dictionary
119
+
120
+ :param sentence: tokenized sentence
121
+ :param labels: labels of form B-short, B-long, I-short, I-long, O
122
+ :return: dictionary
123
+ '''
124
+ shorts = []
125
+ longs = []
126
+ isShort = True
127
+ phr = []
128
+ for i in range(len(sentence)):
129
+ if labels[i] == 'O' or (isShort and 'long' in labels[i]) or (not isShort and 'short' in labels[i]) or (
130
+ labels[i].startswith('B')):
131
+ if len(phr):
132
+ if isShort:
133
+ shorts.append((phr[0], phr[-1]))
134
+ else:
135
+ longs.append((phr[0], phr[-1]))
136
+ phr = []
137
+ if 'short' in labels[i]:
138
+ isShort = True
139
+ phr.append(i)
140
+ if 'long' in labels[i]:
141
+ isShort = False
142
+ phr.append(i)
143
+ if len(phr):
144
+ if isShort:
145
+ shorts.append((phr[0], phr[-1]))
146
+ else:
147
+ longs.append((phr[0], phr[-1]))
148
+ acr_long = {}
149
+ for long in longs:
150
+ best_short = []
151
+ ## check if the long form is already mapped in given diction
152
+ if long in diction and diction[long] in shorts:
153
+ best_short = diction[long]
154
+ best_dist = float('inf')
155
+ #### FIXED [issue: long form incorrectly mapped to the closest acronym in the sentence]
156
+ #### FIXED [issue: multiple short forms could be character matched with the long form]
157
+ if not best_short:
158
+ best_short_cands = []
159
+ for short in shorts:
160
+ long_form = self.character_match(sentence[short[0]], sentence[long[0]:long[1] + 1],
161
+ list(range(long[1] + 1 - long[0])), output_string=True,
162
+ is_candidate=False)
163
+ if long_form:
164
+ best_short_cands.append(short)
165
+ if len(best_short_cands) == 1:
166
+ best_short = best_short_cands[0]
167
+ #####
168
+ #### FIXED [QALD-6 (the workshop of question answering over linked-data 6) at ESWIC 2016]
169
+ if not best_short and map_chars:
170
+ best_short_cands = []
171
+ for short in shorts:
172
+ long_form = self.map_chars(sentence[short[0]], sentence[long[0]:long[1] + 1])
173
+ if long_form:
174
+ best_short_cands.append(short)
175
+ if len(best_short_cands) == 1:
176
+ best_short = best_short_cands[0]
177
+ ####
178
+ #### FIXED [issue: US Securities and Exchange Commission EDGAR ( SEC ) database]
179
+ if not best_short:
180
+ best_short_cands = []
181
+ for short in shorts:
182
+ is_mapped = self.map_chars_with_capitals(sentence[short[0]], sentence[long[0]:long[1] + 1])
183
+ if is_mapped:
184
+ best_short_cands.append(short)
185
+ if len(best_short_cands) == 1:
186
+ best_short = best_short_cands[0]
187
+ ####
188
+ # FIXED [issue: RNNs , Long Short - Term Memory ( LSTM ) architecture]
189
+ if not best_short and long[1] < len(sentence) - 2 and sentence[long[1] + 1] == '(' and 'short' in labels[
190
+ long[1] + 2]:
191
+ for short in shorts:
192
+ if short[0] == long[1] + 2:
193
+ best_short = short
194
+ break
195
+ if not best_short and long[0] > 1 and sentence[long[0] - 1] == '(' and 'short' in labels[long[0] - 2]:
196
+ for short in shorts:
197
+ if short[1] == long[0] - 2:
198
+ best_short = short
199
+ break
200
+ ####
201
+ if not best_short:
202
+ for short in shorts:
203
+ if short[0] > long[1]:
204
+ dist = short[0] - long[1]
205
+ else:
206
+ dist = long[0] - short[1]
207
+ if dist < best_dist:
208
+ best_dist = dist
209
+ best_short = short
210
+ if best_short:
211
+ short_form_info = ' '.join(sentence[best_short[0]:best_short[1] + 1])
212
+ long_form_info = [' '.join(sentence[long[0]:long[1] + 1]), best_short, [long[0], long[1]], tag, 1]
213
+ if short_form_info in acr_long:
214
+ long_form_info[4] += 1
215
+ acr_long[short_form_info] = long_form_info
216
+ if all_acronyms:
217
+ for short in shorts:
218
+ acr = ' '.join(sentence[short[0]:short[1] + 1])
219
+ if acr not in acr_long:
220
+ acr_long[acr] = ['', short, [], tag, 1]
221
+ return acr_long
222
+
223
+ #### FIXED [QALD-6 (the workshop of question answering over linked-data 6) at ESWIC 2016]
224
+ def map_chars(self, acronym, long):
225
+ '''
226
+ This function evaluate the long for based on number of initials overlapping with the acronym and if it is above a threshold it assigns the long form the the acronym
227
+
228
+ :param acronym:
229
+ :param long:
230
+ :return:
231
+ '''
232
+ capitals = []
233
+ for c in acronym:
234
+ if c.isupper():
235
+ capitals.append(c.lower())
236
+ initials = [w[0].lower() for w in long]
237
+ ratio = len([c for c in initials if c in capitals]) / len(initials)
238
+ if ratio >= 0.6:
239
+ return long
240
+ else:
241
+ return None
242
+
243
+ #### FIXED [issue: US Securities and Exchange Commission EDGAR ( SEC ) database]
244
+ def map_chars_with_capitals(self, acronym, long):
245
+ '''
246
+ This function maps the acronym to the long-form which has the same initial capitals as the acronym
247
+
248
+ :param acronym:
249
+ :param long:
250
+ :return:
251
+ '''
252
+ capitals = []
253
+ for c in acronym:
254
+ if c.isupper():
255
+ capitals.append(c.lower())
256
+ long_capital_initials = []
257
+ for w in long:
258
+ if w[0].isupper():
259
+ long_capital_initials.append(w[0].lower())
260
+ if len(capitals) == len(long_capital_initials) and all(
261
+ capitals[i] == long_capital_initials[i] for i in range(len(capitals))):
262
+ return True
263
+ else:
264
+ return False
265
+
266
+ def schwartz_extract(self, sentence, shorts, remove_parentheses, ignore_hyphen=False, ignore_punc=False,
267
+ add_punc=False, small_window=False, no_stop_words=False, ignore_righthand=False,
268
+ map_chars=False,default_diction=False):
269
+ labels = ['O'] * len(sentence)
270
+ diction = {}
271
+ for i, t in enumerate(sentence):
272
+ if i in shorts:
273
+ labels[i] = 'B-short'
274
+ # FIXED [issue: We show that stochastic gradient Markov chain Monte Carlo ( SG - MCMC ) - a class of ]
275
+ if ignore_hyphen:
276
+ t = t.replace('-', '')
277
+ # FIXED [issue: such as Latent Semantic Analysis ( LSA ; )]
278
+ cand_long, cand_long_index, left = self.extract_cand_long(sentence, t, i, ignore_punc=ignore_punc,
279
+ add_punc=add_punc, small_window=small_window)
280
+ cand_long = ' '.join(cand_long)
281
+ long_form = ""
282
+ ## findBestLongForm
283
+ if len(cand_long) > 0:
284
+ if left:
285
+ sIndex = len(t) - 1
286
+ lIndex = len(cand_long) - 1
287
+ while sIndex >= 0:
288
+ curChar = t[sIndex].lower()
289
+ if curChar.isdigit() or curChar.isalpha():
290
+ while (lIndex >= 0 and cand_long[lIndex].lower() != curChar) or (
291
+ sIndex == 0 and lIndex > 0 and (
292
+ cand_long[lIndex - 1].isdigit() or cand_long[lIndex - 1].isalpha())):
293
+ lIndex -= 1
294
+ if lIndex < 0:
295
+ break
296
+ lIndex -= 1
297
+ sIndex -= 1
298
+ if lIndex >= -1:
299
+ try:
300
+ lIndex = cand_long.rindex(" ", 0, lIndex + 1) + 1
301
+ except:
302
+ lIndex = 0
303
+ if cand_long:
304
+ cand_long = cand_long[lIndex:]
305
+ long_form = cand_long
306
+ else:
307
+ sIndex = 0
308
+ lIndex = 0
309
+ if t[0].lower() == cand_long[0].lower() or ignore_righthand:
310
+ while sIndex < len(t):
311
+ curChar = t[sIndex].lower()
312
+ if curChar.isdigit() or curChar.isalpha():
313
+ while (lIndex < len(cand_long) and cand_long[lIndex].lower() != curChar) or (
314
+ ignore_righthand and (sIndex == 0 and lIndex > 0 and (
315
+ cand_long[lIndex - 1].isdigit() or cand_long[lIndex - 1].isalpha()))) or (
316
+ lIndex != 0 and cand_long[lIndex - 1] != ' ' and ' ' in cand_long[
317
+ lIndex:] and
318
+ cand_long[cand_long[lIndex:].index(' ') + lIndex + 1].lower() == curChar):
319
+ lIndex += 1
320
+ if lIndex >= len(cand_long):
321
+ break
322
+ if lIndex >= len(cand_long):
323
+ break
324
+ lIndex += 1
325
+ sIndex += 1
326
+ if lIndex < len(cand_long):
327
+ try:
328
+ lIndex = cand_long[lIndex:].index(" ") + lIndex + 1
329
+ except:
330
+ lIndex = len(cand_long)
331
+ if cand_long:
332
+ cand_long = cand_long[:lIndex]
333
+ long_form = cand_long
334
+ # FIXED [issue : 'good results on the product review ( CR ) and on the question - type ( TREC ) tasks']
335
+ if remove_parentheses:
336
+ if '(' in long_form or ')' in long_form:
337
+ long_form = ''
338
+ # FIXED [issue: TN: The Number of ]
339
+ long_form = long_form.split()
340
+ if no_stop_words and long_form:
341
+ if long_form[0].lower() in stop_words:
342
+ long_form = []
343
+ if long_form:
344
+ if left:
345
+ long_form_index = cand_long_index[-len(long_form):]
346
+ else:
347
+ long_form_index = cand_long_index[:len(long_form)]
348
+ first = True
349
+ for j in range(len(sentence)):
350
+ if j in long_form_index:
351
+ if first:
352
+ labels[j] = 'B-long'
353
+ first = False
354
+ else:
355
+ labels[j] = 'I-long'
356
+ if default_diction:
357
+ diction[(long_form_index[0], long_form_index[-1])] = (i, i)
358
+ return self.create_diction(sentence, labels, tag='Schwartz', map_chars=map_chars, diction=diction)
359
+
360
+ def bounded_schwartz_extract(self, sentence, shorts, remove_parentheses, ignore_hyphen=False, ignore_punc=False,
361
+ add_punc=False, small_window=False, no_stop_words=False, ignore_righthand=False,
362
+ map_chars=False, high_recall=False, high_recall_left=False, tag='Bounded Schwartz',default_diction=False):
363
+ '''
364
+ This function uses the same rule as schwartz but for the format "long form (short form)" will select long forms that the last word in the long form is selected to form the acronym
365
+ example: User - guided Social Media Crawling method ( USMC ) that
366
+
367
+ :param remove_parentheses:
368
+ :param sentence:
369
+ :param shorts:
370
+ :return:
371
+ '''
372
+ labels = ['O'] * len(sentence)
373
+ diction = {}
374
+ for i, t in enumerate(sentence):
375
+ if i in shorts:
376
+ labels[i] = 'B-short'
377
+ # FIXED [issue: We show that stochastic gradient Markov chain Monte Carlo ( SG - MCMC ) - a class of ]
378
+ if ignore_hyphen:
379
+ t = t.replace('-', '')
380
+ # FIXED [issue: The Stopping Trained in America PhDs from Leaving the Economy Act ( or STAPLE Act ) has bee introduced]
381
+ if high_recall:
382
+ cand_long, cand_long_index, left = self.extract_high_recall_cand_long(sentence, t, i,
383
+ small_window=small_window,
384
+ left=high_recall_left)
385
+ else:
386
+ # FIXED [issue: such as Latent Semantic Analysis ( LSA ; )]
387
+ cand_long, cand_long_index, left = self.extract_cand_long(sentence, t, i, ignore_punc=ignore_punc,
388
+ add_punc=add_punc,
389
+ small_window=small_window)
390
+ cand_long = ' '.join(cand_long)
391
+ long_form = ""
392
+ ## findBestLongForm
393
+ if len(cand_long) > 0:
394
+ if left:
395
+ sIndex = len(t) - 1
396
+ lIndex = len(cand_long) - 1
397
+ first_ind = len(cand_long)
398
+ while sIndex >= 0:
399
+ curChar = t[sIndex].lower()
400
+ if curChar.isdigit() or curChar.isalpha():
401
+ while (lIndex >= 0 and cand_long[lIndex].lower() != curChar) or (
402
+ sIndex == 0 and lIndex > 0 and (
403
+ cand_long[lIndex - 1].isdigit() or cand_long[lIndex - 1].isalpha())):
404
+ lIndex -= 1
405
+ if first_ind == len(cand_long):
406
+ first_ind = lIndex
407
+ if lIndex < 0:
408
+ break
409
+ lIndex -= 1
410
+ sIndex -= 1
411
+ if lIndex >= 0 or lIndex == -1 and cand_long[0].lower() == t[0].lower():
412
+ try:
413
+ lIndex = cand_long.rindex(" ", 0, lIndex + 1) + 1
414
+ try:
415
+ rIndex = cand_long[first_ind:].index(" ") + first_ind
416
+ except:
417
+ rIndex = len(cand_long)
418
+ except:
419
+ lIndex = 0
420
+ try:
421
+ rIndex = cand_long[first_ind:].index(" ") + first_ind
422
+ except:
423
+ rIndex = len(cand_long)
424
+ if cand_long:
425
+ index_map = {}
426
+ word_ind = 0
427
+ for ind, c in enumerate(cand_long):
428
+ if c == ' ':
429
+ word_ind += 1
430
+ index_map[ind] = word_ind
431
+ last_word_index = index_map[rIndex - 1]
432
+ cand_long = cand_long[lIndex:rIndex]
433
+ long_form = cand_long
434
+ else:
435
+ sIndex = 0
436
+ lIndex = 0
437
+ first_ind = -1
438
+ if t[0].lower() == cand_long[0].lower() or ignore_righthand:
439
+ while sIndex < len(t):
440
+ curChar = t[sIndex].lower()
441
+ if curChar.isdigit() or curChar.isalpha():
442
+ while (lIndex < len(cand_long) and cand_long[lIndex].lower() != curChar) or (
443
+ ignore_righthand and (sIndex == 0 and lIndex > 0 and (
444
+ cand_long[lIndex - 1].isdigit() or cand_long[lIndex - 1].isalpha()))) or (
445
+ lIndex != 0 and cand_long[lIndex - 1] != ' ' and ' ' in cand_long[
446
+ lIndex:] and
447
+ cand_long[cand_long[lIndex:].index(' ') + lIndex + 1].lower() == curChar):
448
+ lIndex += 1
449
+ if lIndex >= len(cand_long):
450
+ break
451
+ if first_ind == -1:
452
+ first_ind = lIndex
453
+ if lIndex >= len(cand_long):
454
+ break
455
+ lIndex += 1
456
+ sIndex += 1
457
+ if lIndex < len(cand_long) or (
458
+ first_ind < len(cand_long) and lIndex == len(cand_long) and cand_long[-1] == t[-1]):
459
+ try:
460
+ lIndex = cand_long[lIndex:].index(" ") + lIndex + 1
461
+ except:
462
+ lIndex = len(cand_long)
463
+ if cand_long:
464
+ if not ignore_righthand:
465
+ first_ind = 0
466
+ index_map = {}
467
+ word_ind = 0
468
+ for ind, c in enumerate(cand_long):
469
+ if c == ' ':
470
+ word_ind += 1
471
+ index_map[ind] = word_ind
472
+ first_word_index = index_map[first_ind]
473
+ cand_long = cand_long[first_ind:lIndex]
474
+ long_form = cand_long
475
+ # FIXED [issue : 'good results on the product review ( CR ) and on the question - type ( TREC ) tasks']
476
+ if remove_parentheses:
477
+ if '(' in long_form or ')' in long_form:
478
+ long_form = ''
479
+ # FIXED [issue: TN: The Number of ]
480
+ long_form = long_form.split()
481
+ if no_stop_words and long_form:
482
+ if long_form[0].lower() in stop_words:
483
+ long_form = []
484
+ if long_form:
485
+ if left:
486
+ long_form_index = cand_long_index[last_word_index - len(long_form) + 1:last_word_index + 1]
487
+ else:
488
+ long_form_index = cand_long_index[first_word_index:first_word_index + len(long_form)]
489
+ first = True
490
+ for j in range(len(sentence)):
491
+ if j in long_form_index:
492
+ if first:
493
+ labels[j] = 'B-long'
494
+ first = False
495
+ else:
496
+ labels[j] = 'I-long'
497
+ if default_diction:
498
+ diction[(long_form_index[0],long_form_index[-1])] = (i,i)
499
+ return self.create_diction(sentence, labels, tag=tag, map_chars=map_chars,diction=diction)
500
+
501
+ # FIXED [issue: The Stopping Trained in America PhDs from Leaving the Economy Act ( or STAPLE Act ) has bee introduced]
502
+ def high_recall_schwartz(self, sentence, shorts, remove_parentheses, ignore_hyphen=False, ignore_punc=False,
503
+ add_punc=False, small_window=False, no_stop_words=False, ignore_righthand=False,
504
+ map_chars=False):
505
+ '''
506
+ This function use bounded schwartz rules for acronyms which are not necessarily in parentheses
507
+ example: The Stopping Trained in America PhDs from Leaving the Economy Act ( or STAPLE Act ) has bee introduced
508
+
509
+ :param sentence:
510
+ :param shorts:
511
+ :param remove_parentheses:
512
+ :param ignore_hyphen:
513
+ :param ignore_punc:
514
+ :param add_punc:
515
+ :param small_window:
516
+ :param no_stop_words:
517
+ :param ignore_righthand:
518
+ :param map_chars:
519
+ :return:
520
+ '''
521
+ pairs_left = self.bounded_schwartz_extract(sentence, shorts, remove_parentheses, ignore_hyphen=True,
522
+ ignore_punc=ignore_punc, add_punc=add_punc,
523
+ small_window=small_window, no_stop_words=no_stop_words,
524
+ ignore_righthand=ignore_righthand, map_chars=True, high_recall=True,
525
+ high_recall_left=True, tag='High Recall Schwartz')
526
+ pairs_right = self.bounded_schwartz_extract(sentence, shorts, remove_parentheses, ignore_hyphen=True,
527
+ ignore_punc=ignore_punc, add_punc=add_punc,
528
+ small_window=small_window, no_stop_words=no_stop_words,
529
+ ignore_righthand=ignore_righthand, map_chars=True, high_recall=True,
530
+ high_recall_left=False, tag='High Recall Schwartz')
531
+ for acr, lf in pairs_right.items():
532
+ if len(lf[0]) > 0 and (acr not in pairs_left or len(pairs_left[acr][0]) == 0):
533
+ pairs_left[acr] = lf
534
+ res = {}
535
+ for acr, lf in pairs_left.items():
536
+ if acr == ''.join([w[0] for w in lf[0].split() if w[0].isupper()]) or acr.lower() == ''.join(
537
+ w[0] for w in lf[0].split() if w not in string.punctuation and w not in stop_words).lower():
538
+ res[acr] = lf
539
+ return res
540
+
541
+ def character_match(self, acronym, long, long_index, left=False, output_string=False, is_candidate=True):
542
+ capitals = []
543
+ long_form = []
544
+ for c in acronym:
545
+ if c.isupper():
546
+ capitals.append(c)
547
+ # FIXED [issue: different modern GAN architectures : Deep Convolutional ( DC ) GAN , Spectral Normalization ( SN ) GAN , and Spectral Normalization GAN with Gradient Penalty ( SNGP ) .]
548
+ if not is_candidate:
549
+ long_capital_initials = []
550
+ for w in long:
551
+ if w[0].isupper():
552
+ long_capital_initials.append(w[0])
553
+ ####
554
+ if left:
555
+ capitals = capitals[::-1]
556
+ long = long[::-1]
557
+ long_index = long_index[::-1]
558
+ for j, c in enumerate(capitals):
559
+ if j >= len(long):
560
+ long_form = []
561
+ break
562
+ else:
563
+ if long[j][0].lower() == c.lower():
564
+ long_form.append(long_index[j])
565
+ else:
566
+ long_form = []
567
+ break
568
+ # FIXED [issue: different modern GAN architectures : Deep Convolutional ( DC ) GAN , Spectral Normalization ( SN ) GAN , and Spectral Normalization GAN with Gradient Penalty ( SNGP ) .]
569
+ if not is_candidate:
570
+ if len(long_capital_initials) != len(long_form) and len(long_capital_initials) > 0:
571
+ long_form = []
572
+ ####
573
+ long_form.sort()
574
+ if output_string:
575
+ if long_form:
576
+ return long[long_form[0]:long_form[-1] + 1]
577
+ else:
578
+ return ""
579
+ else:
580
+ return long_form
581
+
582
+ # FIXED [issue: annotation software application , Text Annotation Graphs , or TAG , that provides a rich set of]
583
+ def high_recall_character_match(self, sentence, shorts, all_acronyms, ignore_hyphen=False, map_chars=False,default_diction=False):
584
+ '''
585
+ This function finds the long form of the acronyms that are not surrounded by parentheses in the text using scritc rule of character matching (the initial of the sequence of the words in the candidate long form should form the acronym)
586
+ example: annotation software application , Text Annotation Graphs , or TAG , that provides a rich set of ...
587
+
588
+ :param sentence:
589
+ :param shorts:
590
+ :param all_acronyms:
591
+ :return:
592
+ '''
593
+ labels = ['O'] * len(sentence)
594
+ diction = {}
595
+ for i, t in enumerate(sentence):
596
+ if i in shorts:
597
+ labels[i] = 'B-short'
598
+ # FIXED [issue: We show that stochastic gradient Markov chain Monte Carlo ( SG - MCMC ) - a class of ]
599
+ if ignore_hyphen:
600
+ t = t.replace('-', '')
601
+ capitals = []
602
+ for c in t:
603
+ if c.isupper():
604
+ capitals.append(c)
605
+ cand_long = sentence[max(i - len(capitals) - 10, 0):i]
606
+ long_form = ''
607
+ long_form_index = []
608
+ for j in range(max(len(cand_long) - len(capitals), 0)):
609
+ if ''.join(w[0] for w in cand_long[j:j + len(capitals)]) == t:
610
+ long_form = ' '.join(cand_long[j:j + len(capitals)])
611
+ long_form_index = list(range(max(max(i - len(capitals) - 10, 0) + j, 0),
612
+ max(max(i - len(capitals) - 10, 0) + j, 0) + len(capitals)))
613
+ break
614
+ if not long_form:
615
+ cand_long = sentence[i + 1:len(capitals) + i + 10]
616
+ for j in range(max(len(cand_long) - len(capitals), 0)):
617
+ if ''.join(w[0] for w in cand_long[j:j + len(capitals)]) == t:
618
+ long_form = ' '.join(cand_long[j:j + len(capitals)])
619
+ long_form_index = list(range(i + 1 + j, i + j + len(capitals) + 1))
620
+ break
621
+ long_form = long_form.split()
622
+ if long_form:
623
+ if long_form[0] in stop_words or long_form[-1] in stop_words:
624
+ long_form = []
625
+ if any(lf in string.punctuation for lf in long_form):
626
+ long_form = []
627
+ if __name__ != "__main__":
628
+ NPs = [np.text for np in nlp(' '.join(sentence)).noun_chunks]
629
+ long_form_str = ' '.join(long_form)
630
+ if all(long_form_str not in np for np in NPs):
631
+ long_form = []
632
+ if long_form:
633
+ for j in long_form_index:
634
+ labels[j] = 'I-long'
635
+ labels[long_form_index[0]] = 'B-long'
636
+ if default_diction:
637
+ diction[(long_form_index[0], long_form_index[-1])] = (i, i)
638
+ return self.create_diction(sentence, labels, all_acronyms=all_acronyms, tag='high recall character match',
639
+ map_chars=map_chars,diction=diction)
640
+
641
+ def character_match_extract(self, sentence, shorts, all_acronyms, check_all_capitals=False, ignore_hyphen=False,
642
+ ignore_punc=False, map_chars=False,default_diction=False):
643
+ labels = ['O'] * len(sentence)
644
+ diction = {}
645
+ for i, t in enumerate(sentence):
646
+ if i in shorts:
647
+ labels[i] = 'B-short'
648
+ # FIXED [issue: We show that stochastic gradient Markov chain Monte Carlo ( SG - MCMC ) - a class of ]
649
+ if ignore_hyphen:
650
+ t = t.replace('-', '')
651
+ # FIXED [issue: acronyms with lowercase letters, example: of an enhanced Node B ( eNB ) ]
652
+ if check_all_capitals:
653
+ if len(t) != len([c for c in t if c.isupper()]):
654
+ continue
655
+ # FIXED [issue: such as Latent Semantic Analysis ( LSA ; )]
656
+ cand_long, cand_long_index, left = self.extract_cand_long(sentence, t, i, ignore_punc=ignore_punc)
657
+ long_form = []
658
+ if cand_long:
659
+ long_form = self.character_match(t, cand_long, cand_long_index, left, is_candidate=True)
660
+ if long_form:
661
+ labels[long_form[0]] = 'B-long'
662
+ for l in long_form[1:]:
663
+ labels[l] = 'I-long'
664
+ if default_diction:
665
+ diction[(long_form[0], long_form[-1])] = (i, i)
666
+ return self.create_diction(sentence, labels, all_acronyms=all_acronyms, tag='character match',
667
+ map_chars=map_chars, diction=diction)
668
+
669
+ # FIXED [issue: roman numbers]
670
+ def filterout_roman_numbers(self, diction):
671
+ '''
672
+ This function removes roman numbers from the list of extracted acronyms. It removes only numbers from 1 to 20.
673
+ :param diction:
674
+ :return:
675
+ '''
676
+ acronyms = set(diction.keys())
677
+ for acr in acronyms:
678
+ # instead of all roman acronyms we remove only 1 to 20:
679
+ # if bool(re.search(r"^M{0,3}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$", acr)):
680
+ if acr in ['I', 'II', 'III', 'IV', 'V', 'VI', 'VII', 'VIII', 'IX', 'X', 'XI', 'XII', 'XIII', 'XIV', 'XV',
681
+ 'XVI', 'XVII', 'XVIII', 'XIX', 'XX']:
682
+ del diction[acr]
683
+ return diction
684
+
685
+ # FIXED [issue: 'In International Semantic Web Conference , ( ISWC ) ,']
686
+ def remove_punctuations(self, diction):
687
+ '''
688
+ Remove head+tailing punctuations
689
+
690
+ :param diction:
691
+ :return:
692
+ '''
693
+
694
+ for acr, info in diction.items():
695
+ if len(info[0]) > 0:
696
+ if info[0][0] in string.punctuation:
697
+ info[0] = info[0][2:]
698
+ info[2][0] = info[2][0] + 1
699
+ info[3] = 'remove punctuation'
700
+ if len(info[0]) > 0:
701
+ if info[0][-1] in string.punctuation:
702
+ info[0] = info[0][:-2]
703
+ info[2][1] = info[2][1] - 1
704
+ info[3] = 'remove punctuation'
705
+
706
+ return diction
707
+
708
+ # FIXED [issue: and Cantab Capital Institute for Mathematics of Information ( CCIMI )]
709
+ def initial_capitals_extract(self, sentence, shorts, all_acronyms, ignore_hyphen=False, map_chars=False,default_diction=False):
710
+ '''
711
+ This function captures long form which their initials is capital and could form the acronym in the format "long form (acronym)" or "(acronym) long form"
712
+ example:
713
+
714
+ :param sentence:
715
+ :param shorts:
716
+ :param all_acronyms:
717
+ :return:
718
+ '''
719
+ labels = ['O'] * len(sentence)
720
+ diction = {}
721
+ for i, t in enumerate(sentence):
722
+ if i in shorts:
723
+ labels[i] = 'B-short'
724
+ # FIXED [issue: We show that stochastic gradient Markov chain Monte Carlo ( SG - MCMC ) - a class of ]
725
+ if ignore_hyphen:
726
+ t = t.replace('-', '')
727
+ capitals = []
728
+ for c in t:
729
+ if c.isupper():
730
+ capitals.append(c)
731
+ cand_long, cand_long_index, left = self.extract_cand_long(sentence, t, i)
732
+ capital_initials = []
733
+ capital_initials_index = []
734
+ for j, w in enumerate(cand_long):
735
+ lll = labels[i + j - len(cand_long) - 1]
736
+ if w[0].isupper() and labels[i + j - len(cand_long) - 1] == 'O':
737
+ capital_initials.append(w[0])
738
+ capital_initials_index.append(j)
739
+ if ''.join(capital_initials) == t:
740
+ long_form = cand_long[capital_initials_index[0]:capital_initials_index[-1] + 1]
741
+ long_form_index = cand_long_index[capital_initials_index[0]:capital_initials_index[-1] + 1]
742
+ for lfi in long_form_index:
743
+ labels[lfi] = 'I-long'
744
+ labels[long_form_index[0]] = 'B-long'
745
+ if default_diction:
746
+ diction[(long_form_index[0], long_form_index[-1])] = (i, i)
747
+ return self.create_diction(sentence, labels, all_acronyms=all_acronyms, tag='Capital Initials',
748
+ map_chars=map_chars,diction=diction)
749
+
750
+ # FIXED [issue: for C - GAN indicates ]
751
+ def hyphen_in_acronym(self, sentence, shorts):
752
+ '''
753
+ This function merge two acronyms if there is a hyphen between them
754
+ example: for C - GAN indicates
755
+
756
+ :param sentence:
757
+ :param shorts:
758
+ :return:
759
+ '''
760
+
761
+ new_shorts = []
762
+ for short in shorts:
763
+ i = short + 1
764
+ next_hyphen = False
765
+ while i < len(sentence) and sentence[i] == '-':
766
+ next_hyphen = True
767
+ i += 1
768
+ j = short - 1
769
+ before_hyphen = False
770
+ while j > 0 and sentence[j] == '-':
771
+ before_hyphen = True
772
+ j -= 1
773
+ # FIXED [check length of the new acronym. issue: SPG - GCN)In Table]
774
+ # if i < len(sentence) and sentence[i].isupper() and len(sentence[i]) <= 2:
775
+ if i < len(sentence) and sentence[i].isupper() and next_hyphen:
776
+ for ind in range(short + 1, i + 1):
777
+ new_shorts += [ind]
778
+ # FIXED [check length of the new acronym. issue: SPG - GCN)In Table]
779
+ # if j > -1 and sentence[j].isupper() and len(sentence[j]) <= 2:
780
+ if j > -1 and sentence[j].isupper() and before_hyphen:
781
+ for ind in range(j, short):
782
+ new_shorts += [ind]
783
+
784
+ shorts.extend(new_shorts)
785
+ return shorts
786
+
787
+ # FIXED [issue: We show that stochastic gradient Markov chain Monte Carlo ( SG - MCMC ) - a class of ]
788
+ def merge_hyphened_acronyms(self, sentence, labels=[]):
789
+ '''
790
+ This function merge hyphened acronyms
791
+ example: We show that stochastic gradient Markov chain Monte Carlo ( SG - MCMC ) - a class of
792
+
793
+ :param sentence:
794
+ :return:
795
+ '''
796
+ new_sentence = []
797
+ new_labels = []
798
+ merge = False
799
+ shorts = self.short_extract(sentence, 0.6, True)
800
+ shorts += self.hyphen_in_acronym(sentence, shorts)
801
+
802
+ for i, t in enumerate(sentence):
803
+ if i in shorts and i - 1 in shorts and i + 1 in shorts and t == '-':
804
+ merge = True
805
+ if len(new_sentence) > 0:
806
+ new_sentence[-1] += '-'
807
+ else:
808
+ new_sentence += ['-']
809
+ continue
810
+ if merge:
811
+ if len(new_sentence) > 0:
812
+ new_sentence[-1] += t
813
+ else:
814
+ new_sentence += [t]
815
+ else:
816
+ new_sentence.append(t)
817
+ if labels:
818
+ new_labels.append(labels[i])
819
+ merge = False
820
+
821
+ return new_sentence, new_labels
822
+
823
+ # FIXED [issue: we use encoder RNN ( ER )]
824
+ def add_embedded_acronym(self, diction, shorts, sentence):
825
+ '''
826
+ This function will add the embeded acronyms into the dictionary
827
+ example: we use encoder RNN ( ER )
828
+
829
+ :param diction:
830
+ :param shorts:
831
+ :return:
832
+ '''
833
+ short_captured = []
834
+ long_captured = []
835
+ for acr, info in diction.items():
836
+ short_captured.append(info[1][0])
837
+ if info[2]:
838
+ long_captured.extend(list(range(info[2][0], info[2][1])))
839
+ for short in shorts:
840
+ if short not in short_captured and short in long_captured and sentence[short] not in diction:
841
+ diction[sentence[short]] = ['', (short, short), [], 'embedded acronym']
842
+ return diction
843
+
844
+ # FIXED [issue: acronym stands for template]
845
+ def extract_templates(self, sentence, shorts, map_chars=False):
846
+ '''
847
+ Extract acronym and long forms based on templates
848
+ example: PM stands for Product Manager
849
+
850
+ :param sentence:
851
+ :param shorts:
852
+ :return:
853
+ '''
854
+ labels = ['O'] * len(sentence)
855
+ for i, t in enumerate(sentence):
856
+ if i in shorts:
857
+ labels[i] = 'B-short'
858
+ capitals = []
859
+ for c in t:
860
+ if c.isupper():
861
+ capitals.append(c)
862
+ if i < len(sentence) - len(capitals) - 2:
863
+ if sentence[i + 1] == 'stands' and sentence[i + 2] == 'for':
864
+ if ''.join(w[0] for w in sentence[i + 3:i + 3 + len(capitals)]) == ''.join(capitals):
865
+ labels[i + 3:i + 3 + len(capitals)] = ['I-long'] * len(capitals)
866
+ labels[i + 3] = 'B-long'
867
+ return self.create_diction(sentence, labels, all_acronyms=False, tag='Template', map_chars=map_chars)
868
+
869
+ # FIXED [issue: preserve number of meanins extracted from other method]
870
+ def update_pair(self, old_pair, new_pair):
871
+ for acr, info in new_pair.items():
872
+ if acr not in old_pair:
873
+ old_pair[acr] = info
874
+ else:
875
+ info[4] = max(info[4],old_pair[acr][4])
876
+ old_pair[acr] = info
877
+ return old_pair
878
+
879
+ def extract(self, sentence, active_rules):
880
+ # FIXED [issue: of an enhanced Node B ( eNB ) ]
881
+ shorts = self.short_extract(sentence, 0.6, active_rules['starting_lower_case'],
882
+ ignore_dot=active_rules['ignore_dot'])
883
+ # FIXED [issue: acronyms like StESs]
884
+ if active_rules['low_short_threshold']:
885
+ shorts += self.short_extract(sentence, 0.50, active_rules['starting_lower_case'],
886
+ ignore_dot=active_rules['ignore_dot'])
887
+ ####
888
+ # FIXED [issue: for C - GAN indicates ]
889
+ if active_rules['hyphen_in_acronym']:
890
+ shorts += self.hyphen_in_acronym(sentence, shorts)
891
+ ####
892
+ pairs = {}
893
+ if active_rules['schwartz']:
894
+ # FIXED [issue: such as Latent Semantic Analysis ( LSA ; )]
895
+ pairs = self.schwartz_extract(sentence, shorts, active_rules['no_parentheses'],
896
+ ignore_punc=active_rules['ignore_punc_in_parentheses'],
897
+ add_punc=active_rules['extend_punc'],
898
+ small_window=active_rules['small_window'],
899
+ no_stop_words=active_rules['no_beginning_stop_word'],
900
+ ignore_righthand=active_rules['ignore_right_hand'],
901
+ map_chars=active_rules['map_chars'],
902
+ default_diction=active_rules['default_diction'])
903
+ # FIXED [issue: 'User - guided Social Media Crawling method ( USMC ) that']
904
+ if active_rules['bounded_schwartz']:
905
+ # FIXED [issue: such as Latent Semantic Analysis ( LSA ; )]
906
+ bounded_pairs = self.bounded_schwartz_extract(sentence, shorts, active_rules['no_parentheses'],
907
+ ignore_punc=active_rules['ignore_punc_in_parentheses'],
908
+ add_punc=active_rules['extend_punc'],
909
+ small_window=active_rules['small_window'],
910
+ no_stop_words=active_rules['no_beginning_stop_word'],
911
+ ignore_righthand=active_rules['ignore_right_hand'],
912
+ map_chars=active_rules['map_chars'],
913
+ default_diction=active_rules['default_diction'])
914
+ # pairs.update(bounded_pairs)
915
+ pairs = self.update_pair(pairs, bounded_pairs)
916
+ # FIXED [issue: The Stopping Trained in America PhDs from Leaving the Economy Act ( or STAPLE Act ) has bee introduced]
917
+ if active_rules['high_recall_schwartz']:
918
+ hr_paris = self.high_recall_schwartz(sentence, shorts, active_rules['no_parentheses'],
919
+ ignore_punc=active_rules['ignore_punc_in_parentheses'],
920
+ add_punc=active_rules['extend_punc'],
921
+ small_window=active_rules['small_window'],
922
+ no_stop_words=active_rules['no_beginning_stop_word'],
923
+ ignore_righthand=active_rules['ignore_right_hand'],
924
+ map_chars=active_rules['map_chars'],
925
+ default_diction=active_rules['default_diction'])
926
+ # pairs.update(hr_paris)
927
+ pairs = self.update_pair(pairs,hr_paris)
928
+ if active_rules['character']:
929
+ # FIXED [issue: acronyms with lowercase letters, example: of an enhanced Node B ( eNB ) ]
930
+ # FIXED [issue: such as Latent Semantic Analysis ( LSA ; )]
931
+ character_pairs = self.character_match_extract(sentence, shorts, not active_rules['schwartz'],
932
+ check_all_capitals=active_rules['check_all_capitals'],
933
+ ignore_punc=active_rules['ignore_punc_in_parentheses'],
934
+ map_chars=active_rules['map_chars'],
935
+ default_diction=active_rules['default_diction'])
936
+ # pairs.update(character_pairs)
937
+ pairs = self.update_pair(pairs, character_pairs)
938
+ # FIXED [issue: annotation software application , Text Annotation Graphs , or TAG , that provides a rich set of]
939
+ if active_rules['high_recall_character_match']:
940
+ character_pairs = self.high_recall_character_match(sentence, shorts, not active_rules['schwartz'],
941
+ map_chars=active_rules['map_chars'],default_diction=active_rules['default_diction'])
942
+ acronyms = character_pairs.keys()
943
+ for acr in acronyms:
944
+ if acr not in pairs or len(pairs[acr][0]) == 0:
945
+ pairs[acr] = character_pairs[acr]
946
+ # FIXED [issue: and Cantab Capital Institute for Mathematics of Information ( CCIMI )]
947
+ if active_rules['initial_capitals']:
948
+ character_pairs = self.initial_capitals_extract(sentence, shorts, not active_rules['schwartz'],
949
+ map_chars=active_rules['map_chars'],default_diction=active_rules['default_diction'])
950
+ # pairs.update(character_pairs)
951
+ pairs = self.update_pair(pairs,character_pairs)
952
+ # FIXED [issue: acronym stands for long form]
953
+ if active_rules['template']:
954
+ template_pairs = self.extract_templates(sentence, shorts, map_chars=active_rules['map_chars'])
955
+ # pairs.update(template_pairs)
956
+ pairs = self.update_pair(pairs,template_pairs)
957
+ # FIXED [issue: we use encoder RNN ( ER )]
958
+ if active_rules['capture_embedded_acronym']:
959
+ pairs = self.add_embedded_acronym(pairs, shorts, sentence)
960
+ # FIXED [issue: roman numbers]
961
+ if active_rules['roman']:
962
+ pairs = self.filterout_roman_numbers(pairs)
963
+ # FIXED [issue: 'In International Semantic Web Conference , ( ISWC ) ,']
964
+ if active_rules['remove_punctuation']:
965
+ pairs = self.remove_punctuations(pairs)
966
+ return pairs
967
+
968
+ failures = []
969
+ sucess = []
970
+ for i in range(len(gold_label)):
971
+ gold_diction = self.create_diction(dataset[i]['token'], gold_label[i], tag='gold')
972
+ pred_diction = pred_dictions[i]
973
+ if gold_diction.keys() != pred_diction.keys() or set(v[0] for v in gold_diction.values()) != set(
974
+ v[0] for v in pred_diction.values()):
975
+ failures.append([gold_diction, pred_diction, dataset[i]['token'], dataset[i]['id']])
976
+ else:
977
+ sucess.append([gold_diction, pred_diction, dataset[i]['token'], dataset[i]['id']])
978
+ failure_ratio = 'Failures: {:.2%}'.format(len(failures) / len(dataset)) + '\n'
979
+ print(failure_ratio)
980
+ results += failure_ratio
981
+ return failures, sucess, results
popularity.py ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import utils
2
+ import spacy
3
+ from maddog import Extractor
4
+ import constant
5
+ # load
6
+ nlp = spacy.load("en_core_web_sm")
7
+ ruleExtractor = Extractor()
8
+ kb = utils.load_acronym_kb('../input/acronym_kb.json')
9
+
10
+
11
+ def popularity(sentence):
12
+
13
+ tokens = [t.text for t in nlp(sentence) if len(t.text.strip()) > 0]
14
+ rulebased_pairs = ruleExtractor.extract(tokens, constant.RULES)
15
+
16
+ results = list()
17
+ for acronym in rulebased_pairs.keys():
18
+ if rulebased_pairs[acronym][0] != '':
19
+ results.append((acronym, rulebased_pairs[acronym][0]))
20
+ else:
21
+
22
+ pred = utils.get_candidate(kb, acronym, can_num=1)
23
+ results.append((acronym, pred[0]))
24
+ return results
25
+
26
+
27
+ if __name__ == '__main__':
28
+ sentence = \
29
+ "NCBI This new genome assembly and the annotation are tagged as a RefSeq genome by NCBI and thus provide substantially enhanced genomic resources for future research involving S. scovelli."
30
+ results = run_eval(sentence=sentence)
31
+ print(results)
stopWords.txt ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ i
2
+ me
3
+ my
4
+ myself
5
+ we
6
+ our
7
+ ours
8
+ ourselves
9
+ you
10
+ your
11
+ yours
12
+ yourself
13
+ yourselves
14
+ he
15
+ him
16
+ his
17
+ himself
18
+ she
19
+ her
20
+ hers
21
+ herself
22
+ it
23
+ its
24
+ itself
25
+ they
26
+ them
27
+ their
28
+ theirs
29
+ themselves
30
+ what
31
+ which
32
+ who
33
+ whom
34
+ this
35
+ that
36
+ these
37
+ those
38
+ am
39
+ is
40
+ are
41
+ was
42
+ were
43
+ be
44
+ been
45
+ being
46
+ have
47
+ has
48
+ had
49
+ having
50
+ do
51
+ does
52
+ did
53
+ doing
54
+ a
55
+ an
56
+ the
57
+ and
58
+ but
59
+ if
60
+ or
61
+ because
62
+ as
63
+ until
64
+ while
65
+ of
66
+ at
67
+ by
68
+ for
69
+ with
70
+ about
71
+ against
72
+ between
73
+ into
74
+ through
75
+ during
76
+ before
77
+ after
78
+ above
79
+ below
80
+ to
81
+ from
82
+ up
83
+ down
84
+ in
85
+ out
86
+ on
87
+ off
88
+ over
89
+ under
90
+ again
91
+ further
92
+ then
93
+ once
94
+ here
95
+ there
96
+ when
97
+ where
98
+ why
99
+ how
100
+ all
101
+ any
102
+ both
103
+ each
104
+ few
105
+ more
106
+ most
107
+ other
108
+ some
109
+ such
110
+ no
111
+ nor
112
+ not
113
+ only
114
+ own
115
+ same
116
+ so
117
+ than
118
+ too
119
+ very
120
+ s
121
+ t
122
+ can
123
+ will
124
+ just
125
+ don
126
+ should
127
+ now
utils.py ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ This file contains functions for loading various needed data
3
+ """
4
+
5
+ import json
6
+ import torch
7
+ import random
8
+ import logging
9
+ import os
10
+ from random import random as rand
11
+ from torch.utils.data import Dataset
12
+ from torch.utils.data import DataLoader
13
+
14
+ logger = logging.getLogger(__name__)
15
+ local_file = os.path.split(__file__)[-1]
16
+ logging.basicConfig(
17
+ format='%(asctime)s : %(filename)s : %(funcName)s : %(levelname)s : %(message)s',
18
+ level=logging.INFO)
19
+
20
+
21
+ def load_acronym_kb(kb_path='acronym_kb.json'):
22
+ f = open(kb_path, encoding='utf8')
23
+ acronym_kb = json.load(f)
24
+ for key, values in acronym_kb.items():
25
+ values = [v for v, s in values]
26
+ acronym_kb[key] = values
27
+ logger.info('loaded acronym dictionary successfully, in total there are [{a}] acronyms'.format(a=len(acronym_kb)))
28
+ return acronym_kb
29
+
30
+
31
+ def get_candidate(acronym_kb, short_term, can_num=10):
32
+ return acronym_kb[short_term][:can_num]
33
+
34
+ def load_data(path):
35
+ data = list()
36
+ for line in open(path, encoding='utf8'):
37
+ row = json.loads(line)
38
+ data.append(row)
39
+ return data
40
+
41
+
42
+ def load_dataset(data_path):
43
+ all_short_term, all_long_term, all_context = list(), list(), list()
44
+ for line in open(data_path, encoding='utf8'):
45
+ obj = json.loads(line)
46
+ short_term, long_term, context = obj['short_term'], obj['long_term'], ' '.join(obj['tokens'])
47
+ all_short_term.append(short_term)
48
+ all_long_term.append(long_term)
49
+ all_context.append(context)
50
+
51
+ return {'short_term': all_short_term, 'long_term': all_long_term, 'context':all_context}
52
+
53
+
54
+ def load_pretrain(data_path):
55
+ all_short_term, all_long_term, all_context = list(), list(), list()
56
+ cnt = 0
57
+ for line in open(data_path, encoding='utf8'):
58
+ cnt += 1
59
+ # row = line.strip().split('\t')
60
+ # if len(row) != 3:continue
61
+ if cnt>200:continue
62
+ obj = json.loads(line)
63
+ short_term, long_term, context = obj['short_term'], obj['long_term'], ' '.join(obj['tokens'])
64
+ all_short_term.append(short_term)
65
+ all_long_term.append(long_term)
66
+ all_context.append(context)
67
+
68
+ return {'short_term': all_short_term, 'long_term': all_long_term, 'context': all_context}
69
+
70
+
71
+ class TextData(Dataset):
72
+ def __init__(self, data):
73
+ self.all_short_term = data['short_term']
74
+ self.all_long_term = data['long_term']
75
+ self.all_context = data['context']
76
+
77
+ def __len__(self):
78
+ return len(self.all_short_term)
79
+
80
+ def __getitem__(self, idx):
81
+ return self.all_short_term[idx], self.all_long_term[idx], self.all_context[idx]
82
+
83
+
84
+ def random_negative(target, elements):
85
+ flag, result = True, ''
86
+ while flag:
87
+ temp = random.choice(elements)
88
+ if temp != target:
89
+ result = temp
90
+ flag = False
91
+ return result
92
+
93
+
94
+ class SimpleLoader():
95
+ def __init__(self, batch_size, tokenizer, kb, shuffle=True):
96
+ self.batch_size = batch_size
97
+ self.shuffle = shuffle
98
+ self.tokenizer = tokenizer
99
+ self.kb = kb
100
+
101
+ def collate_fn(self, batch_data):
102
+ pos_tag, neg_tag = 0, 1
103
+ batch_short_term, batch_long_term, batch_context = list(zip(*batch_data))
104
+ batch_short_term, batch_long_term, batch_context = list(batch_short_term), list(batch_long_term), list(batch_context)
105
+ batch_negative, batch_label, batch_label_neg = list(), list(), list()
106
+ for index in range(len(batch_short_term)):
107
+ short_term, long_term, context = batch_short_term[index], batch_long_term[index], batch_context[index]
108
+ batch_label.append(pos_tag)
109
+ candidates = [v[0] for v in self.kb[short_term]]
110
+ if len(candidates) == 1:
111
+ batch_negative.append(long_term)
112
+ batch_label_neg.append(pos_tag)
113
+ continue
114
+
115
+ negative = random_negative(long_term, candidates)
116
+ batch_negative.append(negative)
117
+ batch_label_neg.append(neg_tag)
118
+
119
+ prompt = batch_context + batch_context
120
+ long_terms = batch_long_term + batch_negative
121
+ label = batch_label + batch_label_neg
122
+
123
+ encoding = self.tokenizer(prompt, long_terms, return_tensors="pt", padding=True, truncation=True)
124
+ label = torch.LongTensor(label)
125
+
126
+ return encoding, label
127
+
128
+ def __call__(self, data_path):
129
+ dataset = load_dataset(data_path=data_path)
130
+ dataset = TextData(dataset)
131
+ train_iterator = DataLoader(dataset=dataset, batch_size=self.batch_size // 2, shuffle=self.shuffle,
132
+ collate_fn=self.collate_fn)
133
+ return train_iterator
134
+
135
+
136
+ def mask_subword(subword_sequences, prob=0.15, masked_prob=0.8, VOCAB_SIZE=30522):
137
+ PAD, CLS, SEP, MASK, BLANK = 0, 101, 102, 103, -100
138
+ masked_labels = list()
139
+ for sentence in subword_sequences:
140
+ labels = [BLANK for _ in range(len(sentence))]
141
+ original = sentence[:]
142
+ end = len(sentence)
143
+ if PAD in sentence:
144
+ end = sentence.index(PAD)
145
+ for pos in range(end):
146
+ if sentence[pos] in (CLS, SEP): continue
147
+ if rand() > prob: continue
148
+ if rand() < masked_prob: # 80%
149
+ sentence[pos] = MASK
150
+ elif rand() < 0.5: # 10%
151
+ sentence[pos] = random.randint(0, VOCAB_SIZE-1)
152
+ labels[pos] = original[pos]
153
+ masked_labels.append(labels)
154
+ return subword_sequences, masked_labels
155
+
156
+
157
+ class AcroBERTLoader():
158
+ def __init__(self, batch_size, tokenizer, kb, shuffle=True, masked_prob=0.15, hard_num=2):
159
+ self.batch_size = batch_size
160
+ self.shuffle = shuffle
161
+ self.tokenizer = tokenizer
162
+ self.masked_prob = masked_prob
163
+ self.hard_num = hard_num
164
+ self.kb = kb
165
+ self.all_long_terms = list()
166
+ for vs in self.kb.values():
167
+ self.all_long_terms.extend(list(vs))
168
+
169
+ def select_negative(self, target):
170
+ selected, flag, max_time = None, True, 10
171
+ if target in self.kb:
172
+ long_term_candidates = self.kb[target]
173
+ if len(long_term_candidates) == 1:
174
+ long_term_candidates = self.all_long_terms
175
+ else:
176
+ long_term_candidates = self.all_long_terms
177
+ attempt = 0
178
+ while flag and attempt < max_time:
179
+ attempt += 1
180
+ selected = random.choice(long_term_candidates)
181
+ if selected != target:
182
+ flag = False
183
+ if attempt == max_time:
184
+ selected = random.choice(self.all_long_terms)
185
+ return selected
186
+
187
+ def collate_fn(self, batch_data):
188
+ batch_short_term, batch_long_term, batch_context = list(zip(*batch_data))
189
+ pos_samples, neg_samples, masked_pos_samples = list(), list(), list()
190
+ for _ in range(self.hard_num):
191
+ temp_pos_samples = [batch_long_term[index] + ' [SEP] ' + batch_context[index] for index in range(len(batch_long_term))]
192
+ neg_long_terms = [self.select_negative(st) for st in batch_short_term]
193
+ temp_neg_samples = [neg_long_terms[index] + ' [SEP] ' + batch_context[index] for index in range(len(batch_long_term))]
194
+ temp_masked_pos_samples = [batch_long_term[index] + ' [SEP] ' + batch_context[index] for index in range(len(batch_long_term))]
195
+
196
+ pos_samples.extend(temp_pos_samples)
197
+ neg_samples.extend(temp_neg_samples)
198
+ masked_pos_samples.extend(temp_masked_pos_samples)
199
+ return pos_samples, masked_pos_samples, neg_samples
200
+
201
+ def __call__(self, data_path):
202
+ dataset = load_pretrain(data_path=data_path)
203
+ logger.info('loaded dataset, sample = {a}'.format(a=len(dataset['short_term'])))
204
+ dataset = TextData(dataset)
205
+ train_iterator = DataLoader(dataset=dataset, batch_size=self.batch_size // (2 * self.hard_num), shuffle=self.shuffle,
206
+ collate_fn=self.collate_fn)
207
+ return train_iterator
208
+
209
+