|
"""Download important files for the pipeline. Uncomment the following lines if you are running this script for the first time""" |
|
|
|
|
|
|
|
import os |
|
import urllib.request |
|
import tarfile |
|
if not os.path.exists("models/s2v_reddit_2015_md.tar.gz"): |
|
print ("Downloading Sense2Vec model") |
|
urllib.request.urlretrieve(r"https://github.com/explosion/sense2vec/releases/download/v1.0.0/s2v_reddit_2015_md.tar.gz",filename=r"models/s2v_reddit_2015_md.tar.gz") |
|
else: |
|
print ("Sense2Vec model already downloaded") |
|
|
|
reddit_s2v= "models/s2v_reddit_2015_md.tar.gz" |
|
extract_s2v="models" |
|
extract_s2v_folder=reddit_s2v.replace(".tar.gz","") |
|
if not os.path.isdir(extract_s2v_folder): |
|
with tarfile.open(reddit_s2v, 'r:gz') as tar: |
|
tar.extractall(f"models/") |
|
else: |
|
print ("Already extracted") |
|
|
|
"""Import required libraries""" |
|
|
|
import warnings |
|
warnings.filterwarnings('ignore') |
|
|
|
from transformers import T5ForConditionalGeneration,T5Tokenizer |
|
|
|
import streamlit as st |
|
from sense2vec import Sense2Vec |
|
|
|
@st.cache(allow_output_mutation=True) |
|
def cache_models(paths2v,pathT5cond,pathT5): |
|
s2v = Sense2Vec().from_disk(paths2v) |
|
question_model = T5ForConditionalGeneration.from_pretrained(pathT5cond) |
|
question_tokenizer = T5Tokenizer.from_pretrained(pathT5) |
|
return (s2v,question_model,question_tokenizer) |
|
s2v,question_model,question_tokenizer=cache_models("models/s2v_old",'ramsrigouthamg/t5_squad_v1','t5-base') |
|
|
|
|
|
"""Filter out same sense words using sense2vec algorithm""" |
|
|
|
def filter_same_sense_words(original,wordlist): |
|
filtered_words=[] |
|
base_sense =original.split('|')[1] |
|
for eachword in wordlist: |
|
if eachword[0].split('|')[1] == base_sense: |
|
filtered_words.append(eachword[0].split('|')[0].replace("_", " ").title().strip()) |
|
return filtered_words |
|
|
|
def sense2vec_get_words(topn,input_keyword): |
|
word=input_keyword |
|
output=[] |
|
required_keywords=[] |
|
output = [] |
|
try: |
|
sense = s2v.get_best_sense(word) |
|
most_similar = s2v.most_similar(sense, n=topn) |
|
for i in range(len(most_similar)): |
|
required_keywords.append(most_similar[i]) |
|
output = filter_same_sense_words(sense,required_keywords) |
|
print (f"Similar:{output}") |
|
except: |
|
output =[] |
|
|
|
return output |
|
|
|
"""T5 Question generation""" |
|
question_model = T5ForConditionalGeneration.from_pretrained('ramsrigouthamg/t5_squad_v1') |
|
question_tokenizer = T5Tokenizer.from_pretrained('t5-base') |
|
|
|
def get_question(sentence,answer): |
|
text = f"context: {sentence} answer: {answer} </s>" |
|
max_len = 256 |
|
encoding = question_tokenizer.encode_plus(text,max_length=max_len, pad_to_max_length=True, return_tensors="pt") |
|
|
|
input_ids, attention_mask = encoding["input_ids"], encoding["attention_mask"] |
|
|
|
outs = question_model.generate(input_ids=input_ids, |
|
attention_mask=attention_mask, |
|
early_stopping=True, |
|
num_beams=5, |
|
num_return_sequences=1, |
|
no_repeat_ngram_size=2, |
|
max_length=200) |
|
|
|
|
|
dec = [question_tokenizer.decode(ids) for ids in outs] |
|
|
|
|
|
Question = dec[0].replace("question:","") |
|
Question= Question.strip() |
|
return Question |
|
|
