Addind real application

.gitignore

@@ -0,0 +1 @@
+/venv/

app.py

@@ -1,55 +1,54 @@
 import gradio as gr
+from text_converter import generate_similar_sentence
 
-def calc_input_reading_level(input_text):
-    # Placeholder for actual implementation of the reading level calculation.
-    return len(input_text) % 10  # Random operation as a placeholder for demonstration.
-
-def generate_and_analyze_text(input_text, target_level):
-    # Placeholder for generating text and analyzing it against the target level.
-    output_text = input_text[::-1]  # Reversing text as a simple example operation.
-    output_reading_level = int(target_level)  # Placeholder for demonstration.
-    similarity = 0.75  # Fixed similarity value for demonstration.
-    input_level = calc_input_reading_level(input_text)  # Reuse the reading level calc for input level.
-    return input_level, output_text, output_reading_level, similarity
-
-with gr.Blocks() as app:
-    with gr.Row():
-        with gr.Column(scale=1):
-            input_text1 = gr.Textbox(label="Input Text for Reading Level")
-            button1 = gr.Button("Calculate Reading Level", elem_id="button1")
-        with gr.Column(scale=1):
-            input_reading_level = gr.Textbox(label="Input Text Reading Level")
-    
-    with gr.Row():
-        with gr.Column(scale=1):
-            input_text2 = gr.Textbox(label="Input Text for Generation")
-            target_level = gr.Dropdown(choices=["1", "2", "3", "4", "5"], label="Target Reading Level")
-            button2 = gr.Button("Generate and Analyze Text", elem_id="button2")
-        with gr.Column(scale=1):
-            display_input_level = gr.Textbox(label="Input Text Reading Level (Post-Generation)")
-            output_text = gr.Textbox(label="Output Text")
-            output_reading_level = gr.Textbox(label="Output Text Reading Level")
-            output_text_similarity = gr.Textbox(label="Output Text Similarity to Input Text")
-
-    button1.click(
-        fn=calc_input_reading_level,
-        inputs=input_text1,
-        outputs=input_reading_level
-    )
-
-    button2.click(
-        fn=generate_and_analyze_text,
-        inputs=[input_text2, target_level],
-        outputs=[display_input_level, output_text, output_reading_level, output_text_similarity]
-    )
-
-    # Custom CSS to style the buttons
-    app.css = """
-    #button1, #button2 {
-        background-color: orange;
-        color: white;
-        width: 100%;
-    }
-    """
-
-app.launch()
+APP_DESCRIPTION = '''# Reading Level Converter
+<div id="content_align">Convert any text to a specified reading level while retaining the core text meaning</div>'''
+
+MIN_ENTAILMENT = 0.5
+MAX_ITER = 5
+SYSTEM_PROMPT = "You are a writing assistant. You help convert complex texts to simpler texts while maintaining the core meaning of the text."
+
+# Dictionary mapping grade levels to reading ease scores
+reading_levels = {
+    "5th Grade (90-100)": (90, 100),
+    "6th Grade (80-90)": (80, 90),
+    "7th Grade (70-80)": (70, 80),
+    "8th - 9th Grade (60-70)": (60, 70),
+    "10th - 12th Grade (50-60)": (50, 60),
+    "College (30-50)": (30, 50),
+    "College Graduate + Professionals (0-30)": (0, 30)
+}
+
+def convert_text(input_text, grade_level):
+    min_level, max_level = reading_levels[grade_level]
+    output_text, similarity, reading_level, input_reading_level, message = generate_similar_sentence(input_text, min_level, max_level, MIN_ENTAILMENT, SYSTEM_PROMPT, MAX_ITER)
+    return output_text, similarity, reading_level, input_reading_level, message
+
+def main():
+    with gr.Blocks(css='styles.css') as app:
+        gr.Markdown(APP_DESCRIPTION)
+
+        with gr.Tab("Reading Level Calculator"):
+            input_text = gr.Textbox(label="Input Text", placeholder="Type here...", lines=4)
+            grade_level = gr.Radio(choices=list(reading_levels.keys()), label="Target Reading Level", value=list(reading_levels.keys())[0])
+            
+            output_input_reading_level = gr.Textbox(label="Input Text Reading Level", placeholder="Input Text Reading Level...", lines=1)
+            output_reading_level = gr.Textbox(label="Output Reading Level", placeholder="Output Reading Level...", lines=1)
+            output_similarity = gr.Textbox(label="Similarity", placeholder="Similarity Score...", lines=1)
+            output_converted_text = gr.Textbox(label="Converted Text", placeholder="Results will appear here...", lines=4)
+                      
+            output_message = gr.Textbox(label="Message", placeholder="System Message...", lines=2)
+            
+            convert_button = gr.Button("Convert Text")
+
+            convert_button.click(
+                fn=convert_text,
+                inputs=[input_text, grade_level],
+                outputs=[output_converted_text, output_similarity, output_reading_level, output_input_reading_level, output_message]
+            )
+
+    app.launch(inbrowser=True)
+
+
+if __name__ == '__main__':
+    main()

bert_similarity.py

@@ -0,0 +1,76 @@
+from huggingface_hub import from_pretrained_keras
+import tensorflow as tf
+import numpy as np
+import transformers
+
+labels = ["contradiction", "entailment", "neutral"]
+model = from_pretrained_keras("keras-io/bert-semantic-similarity")
+
+class BertSemanticDataGenerator(tf.keras.utils.Sequence):
+    """Generates batches of data."""
+    def __init__(
+        self,
+        sentence_pairs,
+        labels,
+        batch_size=32,
+        shuffle=True,
+        include_targets=True,
+    ):
+        self.sentence_pairs = sentence_pairs
+        self.labels = labels
+        self.shuffle = shuffle
+        self.batch_size = batch_size
+        self.include_targets = include_targets
+        # Load our BERT Tokenizer to encode the text.
+        # We will use base-base-uncased pretrained model.
+        self.tokenizer = transformers.BertTokenizer.from_pretrained(
+            "bert-base-uncased", do_lower_case=True
+        )
+        self.indexes = np.arange(len(self.sentence_pairs))
+        self.on_epoch_end()
+
+    def __len__(self):
+        # Denotes the number of batches per epoch.
+        return len(self.sentence_pairs) // self.batch_size
+
+    def __getitem__(self, idx):
+        # Retrieves the batch of index.
+        indexes = self.indexes[idx * self.batch_size : (idx + 1) * self.batch_size]
+        sentence_pairs = self.sentence_pairs[indexes]
+
+        # With BERT tokenizer's batch_encode_plus batch of both the sentences are
+        # encoded together and separated by [SEP] token.
+        encoded = self.tokenizer.batch_encode_plus(
+            sentence_pairs.tolist(),
+            add_special_tokens=True,
+            max_length=128,
+            truncation=True,
+            return_attention_mask=True,
+            return_token_type_ids=True,
+            pad_to_max_length=True,
+            return_tensors="tf",
+        )
+
+        # Convert batch of encoded features to numpy array.
+        input_ids = np.array(encoded["input_ids"], dtype="int32")
+        attention_masks = np.array(encoded["attention_mask"], dtype="int32")
+        token_type_ids = np.array(encoded["token_type_ids"], dtype="int32")
+
+        # Set to true if data generator is used for training/validation.
+        if self.include_targets:
+            labels = np.array(self.labels[indexes], dtype="int32")
+            return [input_ids, attention_masks, token_type_ids], labels
+        else:
+            return [input_ids, attention_masks, token_type_ids]
+        
+
+
+def get_similarity(sentence1, sentence2):
+    sentence_pairs = np.array([[str(sentence1), str(sentence2)]])
+    test_data = BertSemanticDataGenerator(
+        sentence_pairs, labels=None, batch_size=1, shuffle=False, include_targets=False,
+    )
+    probs = model.predict(test_data[0])[0]
+    
+    labels_probs = {labels[i]: float(probs[i]) for i, _ in enumerate(labels)}
+    return labels_probs['entailment']

styles.css

@@ -0,0 +1,8 @@
+h1 {
+    text-align: center;
+  }
+  
+  #content_align {
+    text-align: center;
+  }
+  

text_converter.py

@@ -0,0 +1,68 @@
+from bert_similarity import get_similarity
+from text_generator import get_gpt_response
+from textstat import flesch_reading_ease
+
+def generate_user_prompt(prompt_type, base_text):
+    prompts = {
+        "too_simple": f"""
+            Convert this text to a higher reading level of the original text.
+            The higher reading level text should have more syllables per word and more words per sentence.
+            It should retain the core meaning of the original text.
+            Here is the text:
+            {base_text}
+        """,
+        "too_complex": f"""
+            Convert this text to a simpler version of the original text.
+            The simpler versions of text have fewer syllables per word and fewer words per sentence.
+            It should retain the core meaning of the original text.
+            Here is the text:
+            {base_text}
+        """
+    }
+
+    return prompts[prompt_type].format(base_text=base_text)
+
+
+def generate_similar_sentence(input_text, min_reading_level, max_reading_level, min_entailment, system_prompt, max_iter):
+    i = 0
+    completed = False
+    user_prompt = ""
+    curr_reading_level = flesch_reading_ease(input_text)
+    input_reading_level = flesch_reading_ease(input_text)
+    curr_text = input_text
+    response = None
+    similarity = 0
+    reading_level = 0
+    print(f"Current reading level is: {curr_reading_level}")
+
+    if curr_reading_level > min_reading_level and curr_reading_level < max_reading_level:
+        return input_text, 1, curr_reading_level, input_reading_level, "Input text was already within the target reading level!"
+
+    else:
+        while i < max_iter and not completed:
+            if curr_reading_level > max_reading_level:
+                print(f"Too simple, current reading level is {curr_reading_level}")
+                user_prompt = generate_user_prompt("too_simple", curr_text)
+            elif curr_reading_level < min_reading_level:
+                print(f"Too complex, current reading level is {curr_reading_level}")
+                user_prompt = generate_user_prompt("too_complex", curr_text)
+            elif similarity < min_entailment:
+                print(f"Entailment level is too low: {similarity}")
+                user_prompt = f"Can you convert this text '{input_text}' to a grade level more similar to this text '{curr_text}'"
+            
+            response = get_gpt_response(user_prompt, system_prompt)
+            similarity = get_similarity(response, input_text)
+            reading_level = flesch_reading_ease(response)
+
+            if similarity >= min_entailment and min_reading_level <= reading_level <= max_reading_level:
+                completed = True
+
+            curr_text = response
+            curr_reading_level = reading_level
+            print(response)
+            i += 1
+
+        if completed:
+            return response, similarity, reading_level, input_reading_level, "Success! Please see the converted text at your target reading level."
+        else:
+            return response, similarity, reading_level, input_reading_level, "Failed. We could not reach the target reading level while maintaining the text meaning."

text_generator.py

@@ -0,0 +1,17 @@
+from openai import OpenAI
+import os
+
+OPENAI_APIKEY = os.environ.get("OPENAI_APIKEY")
+
+client = OpenAI(api_key=OPENAI_APIKEY)
+
+def get_gpt_response(user_prompt, system_prompt):
+    completion = client.chat.completions.create(
+      model="gpt-3.5-turbo",
+      messages=[
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt}
+      ],
+    )
+
+    return completion.choices[0].message.content