Changed random sentence dataset

app.py

@@ -1,8 +1,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: enro-app.ipynb.
 
 # %% auto 0
-__all__ = ['model_name', 'tokenizer', 'model', 'example_translations', 'sentences', 'encoded_key', 'decoded_bytes',
-           'firebase_creds', 'get_translations', 'save_option_to_repo', 'get_random_sentence', 'update_prompt']
+__all__ = ['model_name', 'tokenizer', 'model', 'dataset', 'dialogs', 'encoded_key', 'decoded_bytes', 'firebase_creds',
+           'get_translations', 'clean_sentence', 'get_random_sentence', 'save_option_to_repo', 'update_prompt']
 
 # %% enro-app.ipynb 1
 from transformers import MarianMTModel, MarianTokenizer
@@ -28,60 +28,39 @@ def get_translations(input_text):
     return random_translations[0], random_translations[1]
 
 # %% enro-app.ipynb 5
-example_translations = sentences = [
-    "The mouse jumped.",
-    "The mouse ran away.",
-    "The mouse squeaked loudly.",
-    "The mouse hid quickly.",
-    "The mouse nibbled cheese.",
-    "The mouse dashed off.",
-    "The mouse found crumbs.",
-    "The mouse searched intently.",
-    "The mouse peered out.",
-    "The mouse sniffed carefully.",
-    "The mouse climbed up.",
-    "The mouse scurried along.",
-    "The mouse escaped swiftly.",
-    "The mouse burrowed deeply.",
-    "The mouse explored curiously.",
-    "That mouse squeaked.",
-    "That mouse dashed away.",
-    "That mouse looked around.",
-    "That mouse gnawed wood.",
-    "That mouse hid cleverly.",
-    "That mouse chewed through.",
-    "That mouse fled quickly.",
-    "That mouse found food.",
-    "That mouse evaded capture.",
-    "That mouse peeked out.",
-    "That mouse twitched nervously.",
-    "That mouse made noise.",
-    "That mouse crawled away.",
-    "That mouse gnawed plastic.",
-    "That mouse explored boldly.",
-    "Mice are fast.",
-    "Mice are clever.",
-    "Mice are quick.",
-    "Mice are tiny.",
-    "Mice are playful.",
-    "Mice are resourceful.",
-    "Mice are shy.",
-    "Mice are cute.",
-    "Mice live together.",
-    "Mice hide well.",
-    "Mice avoid predators.",
-    "Mice chew everything.",
-    "Mice burrow underground.",
-    "Mice love food.",
-    "Mice squeak often.",
-    "Mice find crumbs.",
-    "Mice travel far.",
-    "Mice reproduce rapidly.",
-    "Mice build nests.",
-    "Mice hide indoors."
-]
+from datasets import load_dataset
+dataset = load_dataset("daily_dialog")
 
 # %% enro-app.ipynb 6
+import re
+
+dialogs = dataset["train"]
+# Function to clean extra spaces around punctuation marks
+def clean_sentence(sentence):
+    # Remove space before punctuation
+    sentence = re.sub(r'\s+([?.!,"\'-])', r'\1', sentence)
+    # Remove space after punctuation
+    sentence = re.sub(r'([?.!,"\'-])\s+', r'\1 ', sentence)
+    sentence = sentence.strip()
+    return sentence
+
+# Assuming dialogues is a list of lists, where each inner list contains sentences of a dialogue
+# Example: dialogues = [["Hello, how are you?", "I'm fine, thank you!"], ["What's your name?", "My name is John."]]
+
+# Function to randomly select one sentence from the dataset
+def get_random_sentence(dialogs):
+    # Select a random dialogue
+    random_dialogue = random.choice(dialogs['dialog'])
+    # Select a random sentence from the chosen dialogue
+    random_sentence = random.choice(random_dialogue)
+
+    # Clean the sentence
+    clean_random_sentence = clean_sentence(random_sentence)
+    return clean_random_sentence
+
+
+
+# %% enro-app.ipynb 7
 import os
 import firebase_admin
 from firebase_admin import credentials
@@ -98,7 +77,7 @@ if not firebase_admin._apps:
         'databaseURL': 'https://ro-en-llm-default-rtdb.firebaseio.com/'
     })
 
-# %% enro-app.ipynb 7
+# %% enro-app.ipynb 8
 def save_option_to_repo(trans_prompt, translation1, translation2, button):
     ref = db.reference('dpo_feedback')
     # Push new data to the database
@@ -115,8 +94,7 @@ def save_option_to_repo(trans_prompt, translation1, translation2, button):
             'rejected': translation1
         })
 
-def get_random_sentence():
-    return random.sample(example_translations, 1)[0]
+
 
 
 def update_prompt():
@@ -145,7 +123,7 @@ with gr.Blocks() as demo:
         option_buttons.append(gr.Button(value="Translation 2"))
 
     generate.click(get_translations, inputs=prompt, outputs=translations)
-    example_sentence.click(get_random_sentence, outputs=prompt)
+    example_sentence.click(get_random_sentence(dialogs), outputs=prompt)
     for i in range(0,3):
         option_buttons[i].click(save_option_to_repo, inputs=[prompt, translations[0], translations[1], option_buttons[i]])