Create app.py

app.py

@@ -0,0 +1,496 @@
+import gradio as gr
+import tensorflow as tf
+from transformers import TFGPT2LMHeadModel, GPT2Tokenizer
+
+
+tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+
+# add the EOS token as PAD token to avoid warnings
+model = TFGPT2LMHeadModel.from_pretrained("gpt2", pad_token_id=tokenizer.eos_token_id)
+import stanza
+stanza.download('en')
+import nltk
+
+nltk.download('punkt')
+nltk.download('wordnet')
+savejson = True
+indexing = True
+import numpy as np
+import pandas as pd
+from nltk.stem import WordNetLemmatizer
+from nltk.corpus import wordnet
+from nltk.wsd import lesk
+import stanza
+import nltk
+import collections
+import itertools
+import json
+from collections import defaultdict
+nlp = stanza.Pipeline()
+lemmatizer = WordNetLemmatizer()
+stories = pd.read_csv('ROCStories_winter2017 - ROCStories_winter2017 (1).csv')
+import warnings
+warnings.filterwarnings('ignore')
+def defdict():
+    return defaultdict(list)
+class Sentence:
+    def __init__(self, textid, sentencenum, sentence, vectors=None, tfidfs=None, sentenceVector=None, lemmatized=None, preds=None, args=None):
+        self.textid = textid
+        self.sentence = sentence
+        self.sentencenum = sentencenum
+        self.vectors = vectors
+        self.tfidfs = tfidfs
+        self.sentenceVector = sentenceVector
+        self.preds = preds
+        self.args = args
+        if lemmatized:
+            self.lemmatized = lemmatized
+        else:
+            self.lemmatize()
+    
+    def lemmatize(self):
+        doc = nlp(self.sentence)
+        self.lemmatized = []
+        self.preds = []
+        ind = 0
+        self.args=[]
+        for i, dep_edge in enumerate(doc.sentences[0].dependencies):
+            if dep_edge[1] != 'punct':
+                self.lemmatized.append(dep_edge[2].lemma)
+            if dep_edge[1] == "root":
+                self.preds.append(dep_edge[2].lemma)
+                ind = i+1
+        for dep_edge in doc.sentences[0].dependencies:
+            if int(dep_edge[2].head) == ind and dep_edge[1] != 'punct': #dep_edge[1] not in x
+                self.args.append(lemmatizer.lemmatize(dep_edge[2].lemma.lower()))
+        self.doc = doc
+
+    def calculateVector(self):
+        if self.vectors and self.tfidfs:
+            self.sentenceVector = np.dot(self.tfidfs, self.vectors)
+        return self.sentenceVector
+    
+    def getVector(self):
+        if self.sentenceVector is None:
+            self.calculateVector()
+        return self.sentenceVector
+
+class Story:
+    def __init__(self, sentences, number):
+        self.sentences = sentences
+        self.number = number
+    def lemmatizedSents(self):
+        lemSents = []
+        for s in self.sentences:
+            lemSents.append(s.lemmatized)
+        return lemSents
+storiesSentences = []
+sentencesjsons =[]
+def indexSents(sents):
+    ind = defaultdict(defdict)
+    for sc in sents:
+        for i, w in enumerate(sc.lemmatized):
+            ind[w][sc.textid].append((i, sc.sentencenum))
+    return ind
+def indexCorpus():
+    sentences = []
+    #     textid, sentencenum, sentence
+    for i, story in stories[:300].iterrows():
+        storiesSentences.append([])
+        # document = ""
+        print(i)
+        for sind, sent in enumerate(story[2:], start = 1):
+            sentence = Sentence(i, sind-1, sent)
+#            print(sent)
+#            print(i)
+#            print(sentence.sentencenum)
+            # document.join(sent)
+            storiesSentences[i].append(sentence)
+            sentences.append(sentence)
+            sentencesjsons.append(sentence.__dict__)
+    # storiesClasses.append(Story(storiesSentences[i],i))
+    # documents.append(document)
+    return indexSents(sentences)
+if savejson:
+  index = indexCorpus()
+  #json.dump(sentencesjsons, open('filename.json', 'a'))
+else:
+  sentencesjsons = json.load(open('filename.json'))
+# if indexing:
+#   json.dump(index, open('index.json', 'w'))
+# else:
+#   index = json.load(open('index.json'))
+
+def searchByRequest(words):
+    sents = set()
+    dicts = []
+    keys = [] #story numbers
+    synonims = []
+    for i, w in enumerate(words):
+        synonims.append(set())
+        synonims[i].update([w])
+        for synset in wordnet.synsets(w):
+            synonims[i].update(synset.lemma_names())
+    # print(synonims)
+    stories = []
+    dictsForWords = []
+    storiesForWords = []
+    for i, w in enumerate(synonims):
+        dictsForWords.append([])
+        storiesForWords.append(set())
+        for synonim in w:
+            currrentDict = index[synonim]
+            if currrentDict:
+                dictsForWords[i].append(currrentDict)
+                storiesForWords[i].update(set(currrentDict.keys()))
+    paragraphs = set.intersection(*storiesForWords)
+    # print(paragraphs)
+    # print(dictsForWords)
+    # print(dicts)
+    sentencesClasses = set()
+    temporarySentencesByParagraphs = [[set()]*len(words)]*len(paragraphs)
+    for pi, p in enumerate(paragraphs):
+        temporarySentences = []
+        for wi, wordDictsList in enumerate(dictsForWords):
+            temporarySentences.append(set())
+            # print(wordDictsList)
+            for dictionary in wordDictsList:
+                if p in dictionary:
+                    for sents in dictionary[p]:
+                        # print(sents)
+                        temporarySentences[wi].update([sents[1]])
+            #             print(temporarySentences[wi])
+            # print(temporarySentences)
+            if wi>0 and len(words) > 1:
+                for i in range(wi):
+                    for s in temporarySentences[wi]:
+                        if s in temporarySentences[i]:
+                            sentencesClasses.update([storiesSentences[p][s]])
+    # for sentence in sentencesClasses:
+    #     print(sentence.lemmatized)
+    #     print(sentence.sentence, sentence.textid, sentence.sentencenum)
+    return sentencesClasses
+# m = searchByRequest(['play', 'fun', 'game'])
+# m = searchByRequest(['present', 'Christmas', 'wake'])
+def predIndex():
+    # stories = pd.read_csv('ROCStories_winter2017 - ROCStories_winter2017 (1).csv')
+    ind = defaultdict(defdict)
+    for i, story in enumerate(storiesSentences):
+        for j, sent in enumerate(story):
+            for s in sent.preds:
+                ind[s][i].append(j)
+    return ind
+preds = predIndex()
+def powC(subj):
+    c = 0
+    for k, v in preds[subj].items():
+        c += len(v)
+    return c
+def powCons(s1, s2):
+    count = 0
+    for i in (preds[s1].keys() & preds[s2].keys()):
+        i1=0
+        i2=0
+        while i1 != len(preds[s1][i]) and i2 != len(preds[s2][i]): #for d in preds[s1][i]:
+            if preds[s1][i][i1] + 1 == preds[s2][i][i2]:
+                count += 1
+                i1 += 1
+                i2 += 1
+            elif preds[s1][i][i1] + 1 < preds[s2][i][i2]:
+                i1 += 1
+            else:
+                i2 += 1
+    return count
+
+# print(powCons('decide', 'make'))
+# print(powCons('know', 'buy'))
+def synset_lesk(sent, word):
+    sent_tok = nltk.tokenize.word_tokenize(sent)
+    return lesk(sent_tok, word) #,pos
+# comparison of wsd
+# def wpsim():
+# def wpsim_by_max():
+def wpsim_lesk(word1, sent1, word2, sent2):
+    synset1 = lesk(sent1, word1)
+    # print(synset1.definition())
+    synset2 = lesk(sent2, word2)
+    # print(synset2.definition())
+    return synset1.wup_similarity(synset2)
+x = ['punct', 'conj']
+def args_of_pred(s):
+    return s.args
+import math
+alpha = 0.5
+def FRelPred(sent1, sent2):
+  try:
+    p1 = sent1.preds[0]
+    p2 = sent2.preds[0]
+    if powCons(p1, p2) == 0:
+        return 0.0
+    return math.log2(powCons(p1, p2) / (powC(p1)*powC(p2)))
+  except:
+    return 0.2
+def FRelArgs(s1, s2):
+  try:
+    args1 = args_of_pred(s1)
+    args2 = args_of_pred(s2)
+    # print(args1, args2)
+    sent_tok1 = s1.lemmatized#nltk.tokenize.word_tokenize(s1)
+    sent_tok2 = s2.lemmatized#nltk.tokenize.word_tokenize(s2)
+    # print(sent_tok1, sent_tok2)
+    sum1 = 0
+    sum2 = 0
+    max1 = 0
+    max2 = 0
+    wpsim = 0
+    for ni in args1:
+        synsetni = lesk(sent_tok1, ni) #pos
+        # print(synsetni, ni)
+        synsetnj = lesk(sent_tok2, args2[0])
+        # print(synsetnj, args2[0])
+        #         if synsetni != None and synsetnj != None:
+        if not (synsetnj is None or synsetni is None):
+            max1 = synsetni.wup_similarity(synsetnj) #wp_sim(ni, args2[0])
+        # print(type(max1))
+        if max1 is None:
+            max1 = 0
+        for nj in args2[1:]:
+            synsetnj = lesk(sent_tok2, nj)
+            # print(synsetni, ni)
+            # print(synsetnj, nj)
+            #             if synsetni != None and synsetnj != None:
+            if not (synsetnj is None or synsetni is None):
+                wpsim = synsetni.wup_similarity(synsetnj) #(ni, nj)
+            if wpsim is None:
+                wpsim = 0
+            if (not None in [wpsim, max1]) and wpsim > max1:
+                max1 = wpsim
+            # print(wpsim, max1)
+        sum1 += max1
+        # print(sum1)
+    
+    for ni in args2:
+        synsetni = lesk(sent_tok2, ni)
+        synsetnj = lesk(sent_tok1, args1[0])
+        if not (synsetnj is None or synsetni is None):
+            max2 = synsetni.wup_similarity(synsetnj) #wp_sim(ni, args2[0])
+        if max2 is None:
+            max2 = 0
+        for nj in args1[1:]:
+            synsetnj = lesk(sent_tok1, nj)
+            if not (synsetnj is None or synsetni is None):
+                wpsim = synsetni.wup_similarity(synsetnj) #(ni, nj)
+            if wpsim is None:
+                wpsim = 0
+            if (not None in [wpsim, max2]) and wpsim > max2:
+                #             if (wpsim is not None) and wpsim > max2:
+                max2 = wpsim
+        sum2 += max2
+    # print(len(args1))
+    # print(len(args2))
+    # print(sum1, sum2)
+    return 0.5*( (1/len(args1))*sum1 + (1/len(args2))*sum2 )
+  except:
+    return 0.2
+def FRel(s1, s2):
+    return alpha*FRelPred(s1, s2) + (1-alpha)*FRelArgs(s1, s2)
+def hac(foundSentences, length=2):
+    R=0.1
+    twoSentenceClusters = []
+    numfound = len(foundSentences)
+    
+    sentencePairs = []
+    frelijs = []
+    ind = 0
+    maxind = 0
+    maxval = 0
+    for i in itertools.permutations(foundSentences, 2):
+        if i[0].textid != i[1].textid:
+            frelij = FRel(i[0], i[1])
+            if frelij > R:
+                sentencePairs.append(list(i))
+                frelijs.append(frelij)
+                if ind != 0:
+                    if frelij > maxval:
+                        maxval=maxval
+                        maxind = ind
+                        ind += 1
+                else:
+                    ind=1
+                    maxval = frelij
+                    maxind = 0
+    
+    # print(sentencePairs)
+    maxvalThree = 0
+    maxSentsThree = []
+    threeSentsCluster = set()
+    for pairind, pair in enumerate(sentencePairs):
+        for sent in foundSentences:
+            if sent.textid != pair[0].textid and sent.textid != pair[1].textid:
+                frelij = FRel(sent, i[0])
+                if frelij > R:
+                    threeSentsCluster.add(tuple([sent]+pair))
+                    current = (frelijs[maxind]+frelij)/2 > maxvalThree
+                    if current > maxvalThree:
+                        maxvalThree = current
+                        maxSentsThree = [sent]+pair
+                frelij = FRel(i[1], sent)
+                if frelij > R:
+                    threeSentsCluster.add(tuple(pair+[sent]))
+                    current = (frelijs[maxind]+frelij)/2 > maxvalThree
+                    if current > maxvalThree:
+                        maxvalThree = current
+                        maxSentsThree = pair+[sent]
+    # print(sentencePairs)
+    # print(threeSentsCluster)
+    # for pair in sentencePairs:
+    #     print(pair[0].sentence, pair[1].sentence)
+    # for cluster in threeSentsCluster:
+    #     print(cluster[0].sentence, cluster[1].sentence, cluster[2].sentence)
+    # print([sentencePairs[maxind],maxSentsThree])
+    if len(sentencePairs) >=1:
+      return [sentencePairs[maxind],maxSentsThree]#sentencePairs + list(threeSentsCluster)
+    else:
+      return []
+# print(FRelPred('David noticed he had put on a lot of weight recently.',
+#                'He examined his habits to try and figure out the reason.'))
+# #          'After a few weeks, he started to feel much better.'))
+# print(FRel('David noticed he had put on a lot of weight recently.',
+#             'He examined his habits to try and figure out the reason.'))
+# print(FRel('He decided to buy a pair of khakis.', 'The pair he bought fit him perfectly.'))
+from sklearn.feature_extraction.text import TfidfVectorizer
+from gensim.models.word2vec import Word2Vec
+import gensim.downloader
+def tfidfTokenizer(x):
+    return [w for words in [s.lemmatized for s in x] for w in words]
+def preprocess(x):
+    return x
+# tfidfvectorizer = TfidfVectorizer(tokenizer=tfidfTokenizer, preprocessor=preprocess, use_idf=True)
+# tfidfvectorizer_vectors = tfidfvectorizer.fit_transform(storiesSentences) #(map(lambda x: [ s.lemmatized for s in np.array(x).flatten() ] ))
+# wvModel = gensim.downloader.load("word2vec-google-news-300")
+# feature_names = tfidfvectorizer.get_feature_names()
+def setVectors(stories):
+    for doc in stories:
+        for sentence in doc:
+            vectors = []
+            for lemma in sentence.lemmatized:
+                try:
+                    vectors.append(wvModel[lemma])
+                except:
+                    vectors.append([0]*300)
+            sentence.vectors = vectors
+# setVectors(storiesSentences)
+def setTfIdfs(documents):
+    for i, doc in enumerate(documents):
+        feature = tfidfvectorizer_vectors[i,:].nonzero()[1]
+        tfidfs = zip(feature, [tfidfvectorizer_vectors[i, x] for x in feature])
+        tfidfsbyword = dict()
+        for w,s in [(feature_names[j], s) for (j, s) in tfidfs]:
+            tfidfsbyword[w] = s
+        for sent in doc:
+            tfidfs = []
+            for lemma in sent.lemmatized:
+                tfidfs.append(tfidfsbyword[lemma])
+            sent.tfidfs = tfidfs
+            sent.calculateVector()
+# setTfIdfs(storiesSentences)
+# m = searchByRequest(['wake', 'present', 'Christmas'])
+# for sent in m:
+#     print(sent.lemmatized)
+def generate(words):
+    m = searchByRequest(words)
+    return hac(m)
+    
+def generate_and_choose(input_text):
+  input_ids=tokenizer.encode(input_text,return_tensors='tf')
+  beam_outputs=model.generate(input_ids,max_length=100,num_return_sequences=3,num_beams=3,no_repeat_ngram_size=2,early_stopping=True)
+  return_list = []
+  for i, beam_output in enumerate(beam_outputs):
+    # print(beam_output)
+    return_list.append(tokenizer.decode(beam_output, skip_special_tokens=True,clean_up_tokenization_spaces=True))
+  outputs_coherences = []
+  for i, text in enumerate(return_list):
+    sentencesTokenized = nltk.sent_tokenize(text)
+    coherence_cur = 0
+    length = len(sentencesTokenized)
+    for s in range(length-1):
+      coherence_cur += FRel(sentencesTokenized[s], sentencesTokenized[s+1])
+      if length == 1:
+        length += 1
+    outputs_coherences.append(coherence_cur / (length-1))
+  index_of_max = outputs_coherences.index(max(outputs_coherences))
+  return return_list[index_of_max]
+
+def greedy_generate(inp):
+  input_ids = tokenizer.encode(inp, return_tensors='tf')
+  greedy_output = model.generate(input_ids, pad_token_id=tokenizer.encode('.')[0], eos_token_id=tokenizer.encode('.')[0])
+  return tokenizer.decode(greedy_output[0], skip_special_tokens=True)
+
+def with_sampling(input_ids):
+  tf.random.set_seed(0)
+  # activate sampling and deactivate top_k by setting top_k sampling to 0
+  sample_output = model.generate(
+    input_ids, 
+    do_sample=True, 
+    max_length=50, 
+    top_k=0, 
+    temperature=0.7)
+  return tokenizer.decode(sample_output[0], skip_special_tokens=True)
+
+def with_top_k_sampling(input_ids):
+  tf.random.set_seed(0)
+  sample_output = model.generate(
+    input_ids, 
+    do_sample=True, 
+    max_length=50, 
+    top_k=50)
+  return tokenizer.decode(sample_output[0], skip_special_tokens=True)
+
+def with_nucleus_sampling(input_ids):
+  tf.random.set_seed(0)
+  # set top_k = 50 and set top_p = 0.95 and num_return_sequences = 3
+  sample_outputs = model.generate(
+    input_ids,
+    do_sample=True, 
+    max_length=50, 
+    top_k=50, 
+    top_p=0.95, 
+    num_return_sequences=3)
+  return_list = []
+  for i, beam_output in enumerate(sample_outputs):
+    # print(beam_output)
+    return_list.append(tokenizer.decode(beam_output, skip_special_tokens=True,clean_up_tokenization_spaces=True))
+  outputs_coherences = []
+  for i, text in enumerate(return_list):
+    sentencesTokenized = nltk.sent_tokenize(text)
+    coherence_cur = 0
+    length = len(sentencesTokenized)
+    for s in range(length-1):
+      coherence_cur += FRel(sentencesTokenized[s], sentencesTokenized[s+1])
+      if length == 1:
+        length += 1
+    outputs_coherences.append(coherence_cur / (length-1))
+  index_of_max = outputs_coherences.index(max(outputs_coherences))
+  return return_list[index_of_max]
+  
+def generation_method(decoding_algorithm,input_text):
+  input_ids=tokenizer.encode(input_text,return_tensors='tf')
+  if decoding_algorithm=="Beam search":
+    return generate_and_choose(input_text)
+  elif decoding_algorithm=="Greedy search":
+    return greedy_generate(input_text)
+  elif decoding_algorithm=="With sampling":
+    return with_sampling(input_ids)
+  elif decoding_algorithm=="With top k sampling":
+    return with_top_k_sampling(input_ids)
+  elif decoding_algorithm=="With nucleus sampling":
+    return with_nucleus_sampling(input_ids)
+ 
+import gradio as gr
+in1 = gr.inputs.Dropdown(choices=["Beam search", "Greedy search", "With sampling","With top k sampling", "With nucleus sampling"])
+in2 = gr.inputs.Textbox()
+iface = gr.Interface(fn=generation_method, 
+             inputs=[in1,in2],
+             outputs="text").launch(debug=True)
+
+iface.launch()