new commits

celebbot.py

@@ -35,7 +35,7 @@ class CelebBot():
         with sr.Microphone() as mic:
             recognizer.adjust_for_ambient_noise(mic, duration=1)
             # flag = input("Are you ready to record?\nProceed (Y/n)")
-    
+
             # try:
             #     assert flag=='Y'
             # except:
@@ -62,8 +62,8 @@ class CelebBot():
             return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
         # Tokenize sentences
         encoded_input = self.sentTr_tokenizer(texts, padding=True, truncation=True, return_tensors='pt')
-        encoded_input["input_ids"] = encoded_input["input_ids"]
-        encoded_input["attention_mask"] = encoded_input["attention_mask"]
+        encoded_input["input_ids"] = encoded_input["input_ids"].to(self.sentTr_model.device)
+        encoded_input["attention_mask"] = encoded_input["attention_mask"].to(self.sentTr_model.device)
 
         # Compute token embeddings
         with torch.no_grad():
@@ -76,9 +76,9 @@ class CelebBot():
         sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
 
         return sentence_embeddings
-    
+
     def retrieve_knowledge_assertions(self, change_person=True):
-        question_embeddings = self.sentence_embeds_inference([self.name + ', ' + self.text])
+        question_embeddings = self.sentence_embeds_inference([self.text])
 
         all_knowledge_embeddings = self.sentence_embeds_inference(self.all_knowledge)
         similarity = cosine_similarity(all_knowledge_embeddings.cpu(), question_embeddings.cpu())
@@ -89,23 +89,19 @@ class CelebBot():
 
         # similarities = np.array(similarity)[top_K]
 
-        print(*all_knowledge_assertions, sep='\n')
-
-        knowledge = ' '.join(all_knowledge_assertions)
+        # print(*all_knowledge_assertions, sep='\n')
 
         if change_person:
-            return self.third_to_first_person(knowledge)
-        else:
-            return knowledge
-    
+           all_knowledge_assertions = [self.third_to_first_person(sent) for sent in all_knowledge_assertions]
+        return " ".join(all_knowledge_assertions)
+
     def third_to_first_person(self, text):
+        text = text.replace(" ", "  ")
         name = self.name.split(" ")[-1].lower()
         doc = self.spacy_model(text)
         transformed_text = []
 
-        for i, token in enumerate(doc):  
-            if token.text == "pen":
-                print(token.text, token.dep_)
+        for i, token in enumerate(doc):
             if self.gender == "M":
                 if token.text.lower() == "he":
                     transformed_text.append("I")
@@ -114,54 +110,56 @@ class CelebBot():
                 elif token.text.lower() == "his":
                     transformed_text.append("my")
                 elif token.text.lower() == name and token.dep_ in ["nsubj", "nsubjpass"]:
-                    transformed_text.append("I")  
-
-                elif token.text == "'s" and doc[i-1].text.lower() == name:
+                    transformed_text.append("I")
+                elif token.text == "’s" and doc[i-1].text.lower() == name:
                     transformed_text[-1] = "my"
-
+                elif token.text.lower() == "their":
+                    transformed_text.append("our")
+                elif token.text.lower() == "they":
+                    transformed_text.append("we")
                 else:
-                    transformed_text.append(token.text)   
+                    transformed_text.append(token.text)
             elif self.gender == "F":
                 if token.text.lower() == "she":
                     transformed_text.append("I")
                 elif token.text.lower() == "her":
-                    if i != len(doc)-2 and doc[i+1].dep_ in ["nsubj", "nsubjpass", "dobj", "appos", "dative", "attr", "amod"]:
-                        transformed_text.append("my")  
+                    if i != len(doc)-1 and doc[i+2].dep_ in ["nsubj", "nsubjpass", "dobj", "appos", "dative", "attr", "amod", "nummod", "compound", "pobj", "pcomp"]:
+                        transformed_text.append("my")
                     else:
-                        transformed_text.append("me")  
+                        transformed_text.append("me")
                 elif token.text.lower() == name and token.dep_ in ["nsubj", "nsubjpass"]:
-                    transformed_text.append("I")  
-
-                elif token.text == "'s" and doc[i-1].text.lower() == name:
+                    transformed_text.append("I")
+                elif token.text == "’s" and doc[i-1].text.lower() == name:
                     transformed_text[-1] = "my"
-
+                elif token.text.lower() == "their":
+                    transformed_text.append("our")
+                elif token.text.lower() == "they":
+                    transformed_text.append("we")
                 else:
-                    transformed_text.append(token.text)  
+                    transformed_text.append(token.text)
 
-        return " ".join(transformed_text)
+        return "".join(transformed_text)
 
     def question_answer(self, instruction='', knowledge=''):
+        instruction = f"Your name is {self.name}. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."
         if self.text != "":
-            if re.search(re.compile(rf'\b(you|your|ypurs|{self.name})\b', flags=re.IGNORECASE), self.text) != None:              
-                instruction = f"You are a celebrity named {self.name}.  Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."
-                if re.search(re.compile(rf'\b(you|your|yours)\b', flags=re.IGNORECASE), self.text) != None: 
-                    knowledge = self.retrieve_knowledge_assertions()
-                else:
-                    knowledge = self.retrieve_knowledge_assertions(change_person=False)                   
+            if re.search(re.compile(rf'\b(you|your|yours)\b', flags=re.IGNORECASE), self.text) != None:
+              knowledge = self.retrieve_knowledge_assertions()
             else:
-                instruction = f"Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."
+              knowledge = self.retrieve_knowledge_assertions(change_person=False)
+
             query = f"Context: {instruction} {knowledge}\n\nQuestion: {self.text}\n\nAnswer:"
-            input_ids = self.QA_tokenizer(f"{query}", return_tensors="pt").input_ids
-            outputs = self.QA_model.generate(input_ids, max_length=1024, min_length=8,do_sample=True, temperature=0.2, repetition_penalty=1.1)
+            input_ids = self.QA_tokenizer(f"{query}", return_tensors="pt").input_ids.to(self.QA_model.device)
+            outputs = self.QA_model.generate(input_ids, max_length=1024, min_length=8, do_sample=True, temperature=0.2, repetition_penalty=2.5)
             self.text = self.QA_tokenizer.decode(outputs[0], skip_special_tokens=True)
-            
+            # self.text = " ".join([i.text.strip().capitalize() for i in self.spacy_model(self.text).sents])
         return self.text
 
     @staticmethod
     def action_time():
         return f"it's {datetime.datetime.now().time().strftime('%H:%M')}"
 
-    @staticmethod 
+    @staticmethod
     def save_kb(kb, filename):
         with open(filename, "wb") as f:
             pickle.dump(kb, f)

rtvc/synthesizer/utils/cleaners.py

@@ -30,8 +30,8 @@ _alphabet2pronunciation = {
     'g': 'jee',        
     'H': 'eiich',
     'h': 'eiich',   
-    'I': 'eye',
-    'i': 'eye',
+    'I': 'I',
+    'i': 'I',
     'J': 'jay',
     'j': 'jay', 
     'K': 'kay',

run_cli.py

@@ -10,8 +10,7 @@ QA_MODEL_ID = "google/flan-t5-large"
 SENTTR_MODEL_ID = "sentence-transformers/all-mpnet-base-v2"
 
 def main():
-    with open("data.json") as json_file:
-        celeb_data = json.load(json_file)
+    celeb_data = get_celeb_data("data.json")
     message = "Please choose your favorite celebrity from\n"\
               "1. Cate Blanchett\n"\
               "2. David Beckham\n"\
@@ -25,37 +24,23 @@ def main():
     sentTr_tokenizer = AutoTokenizer.from_pretrained(SENTTR_MODEL_ID)
     sentTr_model = AutoModel.from_pretrained(SENTTR_MODEL_ID)
 
-    name = input(message)
-    gender = celeb_data[name]["gender"]
-    knowledge = celeb_data[name]["knowledge"]
-    
-    lname = name.split(" ")[-1]
-    lname_regex = re.compile(rf'\b({lname})\b')
-    name_regex = re.compile(rf'\b({name})\b')
-    lnames = lname+"’s" if not lname.endswith("s") else lname+"’"
-    lnames_regex = re.compile(rf'\b({lnames})\b')
-    names = name+"’s" if not name.endswith("s") else name+"’"
-    names_regex = re.compile(rf'\b({names})\b')
-    if gender == "M":
-        knowledge = re.sub(he_regex, "I", knowledge)
-        knowledge = re.sub(his_regex, "my", knowledge)
-    elif gender == "F":
-        knowledge = re.sub(she_regex, "I", knowledge)
-        knowledge = re.sub(her_regex, "my", knowledge)
-    knowledge = re.sub(names_regex, "my", knowledge)
-    knowledge = re.sub(lnames_regex, "my", knowledge)
-    knowledge = re.sub(name_regex, "I", knowledge)
-    knowledge = re.sub(lname_regex, "I", knowledge)
-    
-    spacy_model = spacy.load("en_core_web_sm")
+    celeb_name = input(message)
+    gender = celeb_data[celeb_name]["gender"]
+    if celeb_name == "Madonna":
+        name = "Madonna-American-singer-and-actress"
+    elif celeb_name == "Anne Hathaway":
+        name = "Anne-Hathaway-American-actress"
+    else:
+        name="-".join(celeb_name.split(" "))
+    knowledge = get_article(f"https://www.britannica.com/biography/{name}")
+    spacy_model = spacy.load("en_core_web_lg")
     knowledge_sents = [i.text.strip() for i in spacy_model(knowledge).sents]
 
-    ai = CelebBot(name, QA_tokenizer, QA_model, sentTr_tokenizer, sentTr_model, spacy_model, knowledge_sents)
+    ai = CelebBot(celeb_name, gender, QA_tokenizer, QA_model, sentTr_tokenizer, sentTr_model, spacy_model, knowledge_sents)
 
     answers = []
     
     while True:
-    # for q in celeb_data[ai.name_id][ai.name]["questions"]:
         if not DEBUG:
             ai.speech_to_text()
         else:

run_eval.py

@@ -9,82 +9,58 @@ import torch
 from utils import *
 from celebbot import CelebBot
 
-QA_MODEL_ID = "google/flan-t5-xl"
+QA_MODEL_ID = "google/flan-t5-large"
 SENTTR_MODEL_ID = "sentence-transformers/all-mpnet-base-v2"
 celeb_names = ["Cate Blanchett", "David Beckham", "Emma Watson", "Lady Gaga", "Madonna", "Mark Zuckerberg"]
 
-def evaluate_system():
+celeb_data = get_celeb_data("data.json")
+references = [val['answers'] for key, val in list(celeb_data.items()) if key in celeb_names]
+references = list(itertools.chain.from_iterable(references))
+predictions = []
 
-    device = 'cpu'
-    with open("data.json", encoding='utf-8') as json_file:
-        celeb_data = json.load(json_file)
-    references = [val['answers'] for key, val in list(celeb_data.items()) if key in celeb_names]
-    references = list(itertools.chain.from_iterable(references))
-    predictions = []
+device = 'cpu'
+QA_tokenizer = AutoTokenizer.from_pretrained(QA_MODEL_ID)
+QA_model = AutoModelForSeq2SeqLM.from_pretrained(QA_MODEL_ID).to(device)
+sentTr_tokenizer = AutoTokenizer.from_pretrained(SENTTR_MODEL_ID)
+sentTr_model = AutoModel.from_pretrained(SENTTR_MODEL_ID).to(device)
 
-    QA_tokenizer = AutoTokenizer.from_pretrained(QA_MODEL_ID)
-    QA_model = AutoModelForSeq2SeqLM.from_pretrained(QA_MODEL_ID).to(device)
-    sentTr_tokenizer = AutoTokenizer.from_pretrained(SENTTR_MODEL_ID)
-    sentTr_model = AutoModel.from_pretrained(SENTTR_MODEL_ID).to(device)
-    
-    for name in celeb_names:
-        gender = celeb_data[name]["gender"]
-        knowledge = celeb_data[name]["knowledge"]
-        
-        lname = name.split(" ")[-1]
-        lname_regex = re.compile(rf'\b({lname})\b')
-        name_regex = re.compile(rf'\b({name})\b')
-        lnames = lname+"’s" if not lname.endswith("s") else lname+"’"
-        lnames_regex = re.compile(rf'\b({lnames})\b')
-        names = name+"’s" if not name.endswith("s") else name+"’"
-        names_regex = re.compile(rf'\b({names})\b')
-        if gender == "M":
-            knowledge = re.sub(he_regex, "I", knowledge)
-            knowledge = re.sub(his_regex, "my", knowledge)
-        elif gender == "F":
-            knowledge = re.sub(she_regex, "I", knowledge)
-            knowledge = re.sub(her_regex, "my", knowledge)
-        knowledge = re.sub(names_regex, "my", knowledge)
-        knowledge = re.sub(lnames_regex, "my", knowledge)
-        knowledge = re.sub(name_regex, "I", knowledge)
-        knowledge = re.sub(lname_regex, "I", knowledge)
-        
-        spacy_model = spacy.load("en_core_web_sm")
-        knowledge_sents = [i.text.strip() for i in spacy_model(knowledge).sents]
+for celeb_name in celeb_names:
+    gender = celeb_data[celeb_name]["gender"]
+    if celeb_name == "Madonna":
+        name = "Madonna-American-singer-and-actress"
+    elif celeb_name == "Anne Hathaway":
+        name = "Anne-Hathaway-American-actress"
+    else:
+        name="-".join(celeb_name.split(" "))
+    knowledge = get_article(f"https://www.britannica.com/biography/{name}")
 
-        ai = CelebBot(name, QA_tokenizer, QA_model, sentTr_tokenizer, sentTr_model, spacy_model, knowledge_sents)
-        if re.search(re.compile(rf'\b(you|your|{ai.name})\b', flags=re.IGNORECASE), ai.text) != None:              
-            instruction1 = f"You are a celebrity named {ai.name}.  Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."
+    spacy_model = spacy.load("en_core_web_lg")
+    knowledge_sents = [i.text.strip() for i in spacy_model(knowledge).sents]
 
-            knowledge = ai.retrieve_knowledge_assertions()
-        else:
-            instruction1 = f"Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."
-        queries = [f"Context: {instruction1} {knowledge}\n\nQuestion: {q}\n\nAnswer:" for q in celeb_data[name]["questions"]]
-        input_ids = ai.QA_tokenizer(f"{queries}", return_tensors="pt").input_ids.to(device)
-        outputs = ai.QA_model.generate(input_ids, max_length=1024)
-        predictions+= ai.QA_tokenizer.batch_decode(outputs, skip_special_tokens=True)
+    ai = CelebBot(celeb_name, gender, QA_tokenizer, QA_model, sentTr_tokenizer, sentTr_model, spacy_model, knowledge_sents)
+    for q in celeb_data[celeb_name]["questions"]:
+        ai.text = q
+        response = ai.question_answer()
+        print("response:", response)
+        predictions.append(response)
 
-    file = open('predictions.txt','w')
-    for prediction in predictions:
-        file.write(prediction+"\n")
-    file.close()
+file = open('predictions.txt','w')
+for prediction in predictions:
+    file.write(prediction+"\n")
+file.close()
 
-    bleu = evaluate.load("bleu")
-    results = bleu.compute(predictions=predictions, references=references, max_order=4)
-    print(f"BLEU: {round(results['bleu'], 2)}")
+bleu = evaluate.load("bleu")
+results = bleu.compute(predictions=predictions, references=references, max_order=4)
+print(f"BLEU: {round(results['bleu'], 2)}")
 
-    meteor = evaluate.load("meteor")
-    results = meteor.compute(predictions=predictions, references=references)
-    print(f"METEOR: {round(results['meteor'], 2)}")
+meteor = evaluate.load("meteor")
+results = meteor.compute(predictions=predictions, references=references)
+print(f"METEOR: {round(results['meteor'], 2)}")
 
-    rouge = evaluate.load("rouge")
-    results = rouge.compute(predictions=predictions, references=references)
-    print(f"ROUGE: {round(results['rougeL'], 2)}")
+rouge = evaluate.load("rouge")
+results = rouge.compute(predictions=predictions, references=references)
+print(f"ROUGE: {round(results['rougeL'], 2)}")
 
-    bertscore = evaluate.load("bertscore")
-    results = bertscore.compute(predictions=predictions, references=references, rescale_with_baseline=True, lang="en")
-    print(f"F1: {round(sum(results['f1'])/len(results['f1']), 2)}")
-    
-if __name__ == "__main__":
-    evaluate_system()
-    
+bertscore = evaluate.load("bertscore")
+results = bertscore.compute(predictions=predictions, references=references, rescale_with_baseline=True, lang="en")
+print(f"F1: {round(sum(results['f1'])/len(results['f1']), 2)}")

run_tts.py

@@ -119,8 +119,8 @@ def tts(text, embed_name, nlp, autoplay=True):
 
 if __name__ == "__main__":
     text = "Adkins was raised by a young single mother in various working-class neighbourhoods of London."
-    embed_name = "Beyonce"
-    nlp = spacy.load('en_core_web_sm')
+    embed_name = "Helen_Mirren"
+    nlp = spacy.load('en_core_web_lg')
     b64 = tts(text, embed_name, nlp, autoplay=False)
 
     md = f"""

utils.py

@@ -36,7 +36,6 @@ def get_celeb_data(fpath):
     with open(fpath, encoding='UTF-8') as json_file:
         return json.load(json_file)
 
-
 def get_article(url):
     req = Request(
     url=url,
@@ -61,7 +60,7 @@ def get_article(url):
         # drop blank lines
         text = ' '.join(chunk for chunk in chunks if chunk)
         return text
-    
+
     except:
         st.markdown("The internet is not stable.")
         return ""
@@ -72,10 +71,5 @@ def get_spacy_model(model_id):
 
 def preprocess_text(name, text:str, model_id):
     spacy_model = get_spacy_model(model_id)
-
-    text = text.replace("’", "'")
-    text = text.replace("‘", "'")
-    text = text.replace("”", "\"")
-    text = text.replace("“", "\"")
     texts = [i.text.strip() for i in spacy_model(text).sents]
     return spacy_model, texts