xtwigs commited on
Commit
2b20905
1 Parent(s): 2fb514b
Files changed (2) hide show
  1. app.py +123 -0
  2. requirements.txt +4 -0
app.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from transformers import AutoTokenizer, AutoModelForSequenceClassification, BartTokenizer, BartForConditionalGeneration, pipeline
3
+ import numpy as np
4
+ import torch
5
+ from textstat import textstat
6
+
7
+
8
+
9
+ MAX_LEN = 256
10
+ NUM_BEAMS = 4
11
+ EARLY_STOPPING = True
12
+ N_OUT = 4
13
+
14
+
15
+
16
+ cwi_tok = AutoTokenizer.from_pretrained('twigs/cwi-regressor')
17
+ cwi_model = AutoModelForSequenceClassification.from_pretrained('twigs/cwi-regressor')
18
+ simpl_tok = BartTokenizer.from_pretrained('twigs/bart-text2text-simplifier')
19
+ simpl_model = BartForConditionalGeneration.from_pretrained('twigs/bart-text2text-simplifier')
20
+ cwi_pipe = pipeline('text-classification', model=cwi_model, tokenizer=cwi_tok, function_to_apply='none', device=0)
21
+ fill_pipe = pipeline('fill-mask', model=simpl_model, tokenizer=simpl_tok, top_k=1, device=0)
22
+
23
+
24
+ def id_replace_complex(s, threshold=0.4):
25
+
26
+ # get all tokens
27
+ tokens = re.compile('\w+').findall(s)
28
+ cands = [f"{t}. {s}" for t in tokens]
29
+ # get complex tokens
30
+ # if score >= threshold select tokens[idx]
31
+ compl_tok = [tokens[idx] for idx, x in enumerate(
32
+ cwi_pipe(cands)) if x['score'] >= threshold]
33
+
34
+ # potentially parallelizable, depends on desired behaviour
35
+ for t in compl_tok:
36
+ idx = s.index(t)
37
+ s = s[:idx] + '<mask>' + s[idx+len(t):]
38
+ # get top candidate for mask fill in complex token
39
+ s = fill_pipe(s)[0]['sequence']
40
+
41
+ return s, compl_tok
42
+
43
+
44
+ def generate_candidate_text(s, model, tokenizer, tokenized=False):
45
+
46
+ out = simpl_tok([s], max_length=256, padding="max_length", truncation=True, return_tensors='pt').to('cuda') if not tokenized else s
47
+
48
+ generated_ids = model.generate(
49
+ input_ids=out['input_ids'],
50
+ attention_mask=out['attention_mask'],
51
+ use_cache=True,
52
+ decoder_start_token_id=simpl_model.config.pad_token_id,
53
+ num_beams=NUM_BEAMS,
54
+ max_length=MAX_LEN,
55
+ early_stopping=EARLY_STOPPING,
56
+ num_return_sequences=N_OUT
57
+ )
58
+
59
+ return [tokenizer.decode(ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)[
60
+ 1:] for ids in generated_ids]
61
+
62
+
63
+ def rank_candidate_text(sentences):
64
+ """ Currently being done with simple FKGL """
65
+ fkgl_scores = [textstat.flesch_kincaid_grade(s) for s in sentences]
66
+ return sentences[np.argmin(fkgl_scores)]
67
+
68
+
69
+ def full_pipeline(source, simpl_model, simpl_tok, tokens, lexical=False):
70
+
71
+ modified, complex_words = id_replace_complex(source, threshold=0.2) if lexical else source, None
72
+ cands = generate_candidate_text(tokens+modified, simpl_model, simpl_tok)
73
+ output = rank_candidate_text(cands)
74
+ return output, complex_words
75
+
76
+
77
+ aug_tok = ['c_', 'lev_', 'dep_', 'rank_', 'rat_', 'n_syl_']
78
+ tokens = ['CharRatio', 'LevSim', 'DependencyTreeDepth',
79
+ 'WordComplexity', 'WordRatio']
80
+
81
+ default_values = [0.8, 0.6, 0.9, 0.8, 0.9, 1.9]
82
+ user_values = default_values
83
+ tok_values = dict((t, default_values[idx]) for idx, t in enumerate(tokens))
84
+
85
+ example_sentences = ["A matchbook is a small cardboard folder (matchcover) enclosing a quantity of matches and having a coarse striking surface on the exterior.",
86
+ "If there are no strong land use controls, buildings are built along a bypass, converting it into an ordinary town road, and the bypass may eventually become as congested as the local streets it was intended to avoid.",
87
+ "Plot Captain Caleb Holt (Kirk Cameron) is a firefighter in Albany, Georgia and firmly keeps the cardinal rule of all firemen, \"Never leave your partner behind\".",
88
+ "Britpop emerged from the British independent music scene of the early 1990s and was characterised by bands influenced by British guitar pop music of the 1960s and 1970s."]
89
+
90
+
91
+ def main():
92
+
93
+ st.title("Make it Simple")
94
+
95
+ with st.expander("Example sentences"):
96
+ for s in example_sentences:
97
+ st.code(body=s)
98
+
99
+ with st.form(key="form"):
100
+ input_sentence = st.text_area("Original sentence")
101
+ tok = st.multiselect(
102
+ label="Tokens to augment the sentence", options=tokens, default=tokens)
103
+ if (tok):
104
+ st.text("Select the desired intensity")
105
+ for idx, t in enumerate(tok):
106
+ user_values[idx] = st.slider(
107
+ t, min_value=0., max_value=1., value=tok_values[t], step=0.1, key=t)
108
+
109
+ submit = st.form_submit_button("Process")
110
+ if (submit):
111
+
112
+ tokens = [t+str(v) for t, v in zip(aug_tok, user_values)]
113
+ output, words = full_pipeline(input_sentence, simpl_model, simpl_tok, tokens)
114
+
115
+ with st.container():
116
+ st.write("Original sentence:")
117
+ st.write(input_sentence)
118
+ st.write("Output sentence:")
119
+ st.write(output)
120
+
121
+
122
+ if __name__ == '__main__':
123
+ main()
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ transformers
2
+ torch
3
+ numpy
4
+ textstat