feat: release

app.py

@@ -0,0 +1,29 @@
+from generateDistractors.senseToVec import S2V
+from keyExtractor.rake import KeyExtractor
+from questionGeneration.questionGeneration import QuestionGeneration
+from summarizer.summarizer import Summarizer
+import gradio as gr
+
+sense2Vec = S2V()
+Key = KeyExtractor()
+Question = QuestionGeneration()
+Summary = Summarizer()
+
+def run(text):
+  result = []
+  summarized_text = Summary.summarizer(text)
+  imp_keywords = Key.get_keywords(text)
+
+  for answer in imp_keywords:
+    ques = Question.get_question(summarized_text,answer)
+    distractors = sense2Vec.execute(answer)
+    result.append({
+        "question": ques,
+        "answer": answer.capitalize(),
+        "distractors": distractors
+    })
+  return result
+
+if __name__ == '__main__':
+    demo = gr.Interface(fn=run, inputs="text", outputs="json")
+    demo.launch()

generateDistractors/Dockerfile

@@ -0,0 +1,7 @@
+# syntax=docker/dockerfile:1
+FROM python:3.9
+WORKDIR /code
+COPY ./requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+COPY . /code
+CMD ["uvicorn", "senseToVec:app", "--host", "0.0.0.0", "--port", "1237"]

generateDistractors/mmr.py

@@ -0,0 +1,57 @@
+from typing import List, Tuple
+import itertools
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+
+#Maximal Marginal Relevance origin: https://maartengr.github.io/KeyBERT/api/mmr.html
+def mmr(doc_embedding: np.ndarray,
+        word_embeddings: np.ndarray,
+        words: List[str],
+        top_n: int = 5,
+        diversity: float = 0.9) -> List[Tuple[str, float]]:
+    """ Calculate Maximal Marginal Relevance (MMR)
+    between candidate keywords and the document.
+
+
+    MMR considers the similarity of keywords/keyphrases with the
+    document, along with the similarity of already selected
+    keywords and keyphrases. This results in a selection of keywords
+    that maximize their within diversity with respect to the document.
+
+    Arguments:
+        doc_embedding: The document embeddings
+        word_embeddings: The embeddings of the selected candidate keywords/phrases
+        words: The selected candidate keywords/keyphrases
+        top_n: The number of keywords/keyhprases to return
+        diversity: How diverse the select keywords/keyphrases are.
+                   Values between 0 and 1 with 0 being not diverse at all
+                   and 1 being most diverse.
+
+    Returns:
+         List[Tuple[str, float]]: The selected keywords/keyphrases with their distances
+
+    """
+
+    # Extract similarity within words, and between words and the document
+    word_doc_similarity = cosine_similarity(word_embeddings, doc_embedding)
+    word_similarity = cosine_similarity(word_embeddings)
+
+    # Initialize candidates and already choose best keyword/keyphras
+    keywords_idx = [np.argmax(word_doc_similarity)]
+    candidates_idx = [i for i in range(len(words)) if i != keywords_idx[0]]
+
+    for _ in range(top_n - 1):
+        # Extract similarities within candidates and
+        # between candidates and selected keywords/phrases
+        candidate_similarities = word_doc_similarity[candidates_idx, :]
+        target_similarities = np.max(word_similarity[candidates_idx][:, keywords_idx], axis=1)
+
+        # Calculate MMR
+        mmr = (1-diversity) * candidate_similarities - diversity * target_similarities.reshape(-1, 1)
+        mmr_idx = candidates_idx[np.argmax(mmr)]
+
+        # Update keywords & candidates
+        keywords_idx.append(mmr_idx)
+        candidates_idx.remove(mmr_idx)
+
+    return [(words[idx], round(float(word_doc_similarity.reshape(1, -1)[0][idx]), 4)) for idx in keywords_idx]

generateDistractors/readme

@@ -0,0 +1,2 @@
+!wget https://github.com/explosion/sense2vec/releases/download/v1.0.0/s2v_reddit_2015_md.tar.gz
+!tar -xvf s2v_reddit_2015_md.tar.gz

generateDistractors/requirements.txt

@@ -0,0 +1,5 @@
+sense2vec==2.0.1
+sentence_transformers==2.2.2
+pydantic
+fastapi
+uvicorn

generateDistractors/senseToVec.py

@@ -0,0 +1,58 @@
+from sense2vec import Sense2Vec
+from fastapi import FastAPI
+from sentence_transformers import SentenceTransformer
+import wget
+import os
+from .mmr import mmr
+
+url = 'https://github.com/explosion/sense2vec/releases/download/v1.0.0/s2v_reddit_2015_md.tar.gz'
+cmd = 'tar -xvf {}'
+
+class S2V:
+  def __init__(self):
+    self.model= SentenceTransformer('all-MiniLM-L12-v2')
+    filename = wget.download(url)
+    os.system(cmd.format(filename))
+    self.s2v = Sense2Vec().from_disk('s2v_old')
+    
+  def removeDuplicates(self, most_similar, originalword):
+    distractors = []
+    #remove duplicates
+    for each_word in most_similar:
+      append_word = each_word[0].split("|")[0].replace("_", " ")
+      if append_word not in distractors and append_word != originalword:
+          distractors.append(append_word)
+    return distractors
+  
+  def get_answer_and_distractor_embeddings(self,answer,candidate_distractors):
+    answer_embedding = self.model.encode([answer])
+    distractor_embeddings = self.model.encode(candidate_distractors)
+    return answer_embedding,distractor_embeddings
+  
+  def execute(self, originalword):
+    word = originalword.lower()
+    word = word.replace(" ", "_")
+    # Find the best-matching sense for a given word based on the available senses and frequency counts. 
+    sense = self.s2v.get_best_sense(word)
+    # Get the most similar entries in the table
+    most_similar = self.s2v.most_similar(sense, n=20)
+    #remove duplicates
+    distractors = self.removeDuplicates(most_similar, originalword)
+    distractors.insert(0,originalword)
+    # encode distractors and answer
+    answer_embedd, distractor_embedds = self.get_answer_and_distractor_embeddings(originalword,distractors)
+    #Maximal Marginal Relevance origin: https://maartengr.github.io/KeyBERT/api/mmr.html
+    final_distractors = mmr(answer_embedd,distractor_embedds,distractors,5)
+    filtered_distractors = []
+
+    for dist in final_distractors:
+      filtered_distractors.append(dist[0])
+
+    Answer = filtered_distractors[0]
+    Filtered_Distractors =  filtered_distractors[1:]
+    return {
+      "answer": Answer,
+      "distractors": Filtered_Distractors
+    }
+
+sense2Vec = S2V()

keyExtractor/Dockerfile

@@ -0,0 +1,7 @@
+# syntax=docker/dockerfile:1
+FROM python:3.9
+WORKDIR /code
+COPY ./requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+COPY . /code
+CMD ["uvicorn", "rake:app", "--host", "0.0.0.0", "--port", "1234"]

keyExtractor/rake.py

@@ -0,0 +1,19 @@
+from rake_nltk import Rake
+import nltk
+nltk.download('stopwords')
+
+# Uses stopwords for english from NLTK, and all puntuation characters by
+# default
+class KeyExtractor:
+    def __init__(self):
+        self.model = Rake()
+    def get_keywords(self, text):
+        # Extraction given the text.
+        self.model.extract_keywords_from_text(text)
+        # To get keyword phrases ranked highest to lowest.
+        imp_keywords = self.model.get_ranked_phrases()[0:4]
+
+        result = []
+        for answer in imp_keywords:
+            result.append(answer)
+        return result

keyExtractor/requirements.txt

@@ -0,0 +1,4 @@
+pydantic
+fastapi
+uvicorn
+rake-nltk

questionGeneration/Dockerfile

@@ -0,0 +1,7 @@
+# syntax=docker/dockerfile:1
+FROM python:3.9
+WORKDIR /code
+COPY ./requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+COPY . /code
+CMD ["uvicorn", "questionGeneration:app", "--host", "0.0.0.0", "--port", "1236"]

questionGeneration/questionGeneration.py

@@ -0,0 +1,36 @@
+import torch
+from transformers import T5ForConditionalGeneration,T5Tokenizer
+
+class QuestionGeneration:
+    def __init__(self):
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model = T5ForConditionalGeneration.from_pretrained('ramsrigouthamg/t5_squad_v1')
+        self.tokenizer = T5Tokenizer.from_pretrained('ramsrigouthamg/t5_squad_v1')
+        self.model = self.model.to(self.device)
+
+    def get_question(self, context, answer, model = None, tokenizer = None):
+        if(model == None):
+            model = self.model
+        if(tokenizer == None):
+            tokenizer = self.tokenizer
+        text = "context: {} answer: {}".format(context,answer)
+        encoding = tokenizer.encode_plus(text,max_length=384, pad_to_max_length=False,truncation=True, return_tensors="pt").to(self.device)
+        input_ids, attention_mask = encoding["input_ids"], encoding["attention_mask"]
+
+        outs = model.generate(input_ids=input_ids,
+            attention_mask=attention_mask,
+            early_stopping=True,
+            num_beams=5,
+            num_return_sequences=1,
+            no_repeat_ngram_size=2,
+            max_length=72
+        )
+
+        dec = [tokenizer.decode(ids,skip_special_tokens=True) for ids in outs]
+
+
+        Question = dec[0].replace("question:","")
+        Question= Question.strip()
+        return Question
+
+Question = QuestionGeneration()

questionGeneration/requirements.txt

@@ -0,0 +1,6 @@
+torch
+fastapi
+pydantic
+sentencepiece==0.1.95
+transformers
+uvicorn

requirements.txt

@@ -0,0 +1,8 @@
+gradio
+rake-nltk
+sense2vec==2.0.1
+sentence_transformers==2.2.2
+torch
+sentencepiece==0.1.95
+transformers
+nltk

summarizer/Dockerfile

@@ -0,0 +1,7 @@
+# syntax=docker/dockerfile:1
+FROM python:3.9
+WORKDIR /code
+COPY ./requirements.txt /code/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+COPY . /code
+CMD ["uvicorn", "summarizer:app", "--host", "0.0.0.0", "--port", "1235"]

summarizer/requirements.txt

@@ -0,0 +1,7 @@
+torch
+pydantic
+fastapi
+sentencepiece==0.1.95
+transformers
+nltk
+uvicorn

summarizer/summarizer.py

@@ -0,0 +1,70 @@
+import torch
+from transformers import T5ForConditionalGeneration,T5Tokenizer
+import random
+import numpy as np
+import nltk
+
+nltk.download('punkt')
+nltk.download('brown')
+nltk.download('wordnet')
+
+from nltk.corpus import wordnet as wn
+from nltk.tokenize import sent_tokenize
+
+import locale
+locale.getpreferredencoding = lambda: "UTF-8"
+
+class Summarizer:
+    def __init__(self):
+        self.model = T5ForConditionalGeneration.from_pretrained('t5-base')
+        self.tokenizer = T5Tokenizer.from_pretrained('t5-base')
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model = self.model.to(self.device)
+        self.set_seed(42)
+
+    def set_seed(self, seed: int):
+        random.seed(seed)
+        np.random.seed(seed)
+        torch.manual_seed(seed)
+        torch.cuda.manual_seed_all(seed)
+
+    def postprocesstext(self, content):
+        final=""
+        for sent in sent_tokenize(content):
+            sent = sent.capitalize()
+            final = final +" "+sent
+        return final
+
+
+    def summarizer(self, text, model = None, tokenizer = None):
+        if(model == None):
+            model = self.model
+        if(tokenizer == None):
+            tokenizer = self.tokenizer
+        text = text.strip().replace("\n"," ")
+        text = "summarize: "+text
+        max_len = 512
+        encoding = tokenizer.encode_plus(text,max_length=max_len, pad_to_max_length=False,truncation=True, return_tensors="pt").to(self.device)
+
+        input_ids, attention_mask = encoding["input_ids"], encoding["attention_mask"]
+
+        outs = model.generate(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            early_stopping=True,
+            num_beams=3,
+            num_return_sequences=1,
+            no_repeat_ngram_size=2,
+            min_length = 75,
+            max_length=300
+        )
+
+        dec = [tokenizer.decode(ids,skip_special_tokens=True) for ids in outs]
+        summary = dec[0]
+        summary = self.postprocesstext(summary)
+        summary= summary.strip()
+
+        return summary
+
+Summary = Summarizer()
+

testers/bleu-4.py

@@ -0,0 +1,15 @@
+from nltk.translate.bleu_score import sentence_bleu
+
+reference = [
+    'this is a dog'.split(),
+]
+
+candidate = 'this is dog'.split()
+
+# quanto la frase candidata è vicina a quelle di riferimento
+print('Individual 1-gram: %f' % sentence_bleu(reference, candidate, weights=(1, 0, 0, 0)))
+print('Individual 2-gram: %f' % sentence_bleu(reference, candidate, weights=(0, 1, 0, 0)))
+print('Individual 3-gram: %f' % sentence_bleu(reference, candidate, weights=(0, 0, 1, 0)))
+print('Individual 4-gram: %f' % sentence_bleu(reference, candidate, weights=(0, 0, 0, 1)))
+
+print('average 4-gram: %f' % sentence_bleu(reference, candidate, weights=(0.25, 0.25, 0.25, 0.25)))

testers/meteor.py

@@ -0,0 +1,14 @@
+from nltk.translate import meteor
+from nltk import word_tokenize
+import nltk
+
+nltk.download('punkt')
+nltk.download('wordnet')
+
+#calcola una media tra precision e recall con maggiore enfasi su recall
+score = meteor(
+[word_tokenize('create or update a vm set')],
+word_tokenize('creates or updates a virtual machine scale set')
+)
+
+print(f"meteor score: {score}")

testers/rouge-tester.py

@@ -0,0 +1,19 @@
+from rouge import Rouge 
+
+hypothesis = "this is a dog"
+
+reference = "this is a dog"
+
+rouge = Rouge()
+scores = rouge.get_scores(hypothesis, reference, avg=True)
+
+for rouge_type in scores.keys():
+    print(rouge_type)
+    for score in scores[rouge_type]:
+        if(score == 'r'):
+            print(f"recall: {scores[rouge_type][score]}")
+        if(score == 'p'):
+            print(f"precision: {scores[rouge_type][score]}")
+        if(score == 'f'):
+            print(f"f1_score: {scores[rouge_type][score]}")
+    print()