Update pages/2_phonics.py
Browse files- pages/2_phonics.py +414 -414
pages/2_phonics.py
CHANGED
@@ -1,414 +1,414 @@
|
|
1 |
-
# app.py
|
2 |
-
|
3 |
-
import os
|
4 |
-
import json
|
5 |
-
import streamlit as st
|
6 |
-
from PIL import Image
|
7 |
-
import google.generativeai as genai
|
8 |
-
import ast
|
9 |
-
#from utils import findImg
|
10 |
-
import io
|
11 |
-
from streamlit_TTS import auto_play
|
12 |
-
import torch
|
13 |
-
from transformers import pipeline
|
14 |
-
from datasets import load_dataset
|
15 |
-
import soundfile as sf
|
16 |
-
from gtts import gTTS
|
17 |
-
import io
|
18 |
-
from mistralai.models.chat_completion import ChatMessage
|
19 |
-
from mistralai.client import MistralClient
|
20 |
-
from audiorecorder import audiorecorder
|
21 |
-
import base64
|
22 |
-
###
|
23 |
-
import os
|
24 |
-
import cv2
|
25 |
-
import numpy as np
|
26 |
-
from sklearn.metrics.pairwise import cosine_similarity
|
27 |
-
from sentence_transformers import SentenceTransformer
|
28 |
-
from diffusers import StableDiffusionPipeline
|
29 |
-
import torch
|
30 |
-
|
31 |
-
import streamlit as st
|
32 |
-
def add_logo():
|
33 |
-
st.markdown(
|
34 |
-
"""
|
35 |
-
<style>
|
36 |
-
[data-testid="stSidebarNav"] {
|
37 |
-
background-image: url(http://placekitten.com/200/200);
|
38 |
-
background-repeat: no-repeat;
|
39 |
-
#padding-top: 120px;
|
40 |
-
background-position: 20px 20px;
|
41 |
-
}
|
42 |
-
[data-testid="stSidebarNav"]::before {
|
43 |
-
content: "MO3ALIMI sidebar";
|
44 |
-
margin-left: 20px;
|
45 |
-
margin-top: 20px;
|
46 |
-
font-size: 29px;
|
47 |
-
position: relative;
|
48 |
-
top: 0px;
|
49 |
-
}
|
50 |
-
</style>
|
51 |
-
""",
|
52 |
-
unsafe_allow_html=True,
|
53 |
-
)
|
54 |
-
add_logo()
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
device = "cuda" if torch.cuda.is_available() else "cpu"
|
59 |
-
|
60 |
-
|
61 |
-
if 'pipe' not in st.session_state:
|
62 |
-
st.session_state['pipe'] = pipeline("automatic-speech-recognition", model="openai/whisper-large-v3")
|
63 |
-
|
64 |
-
pipe = st.session_state['pipe']
|
65 |
-
|
66 |
-
|
67 |
-
# Set up the API key for Generative AI
|
68 |
-
os.environ["GEMINI_API_KEY"] = "AIzaSyBYZ_7geqmnK6xrSe268-1nSLeuEwbzmTA"
|
69 |
-
|
70 |
-
# Initial prompt to send to the model
|
71 |
-
initial_prompt = """
|
72 |
-
you're an Literacy Instructor for Illiterate Adults
|
73 |
-
you're objective is to Teach illiterate adults how to read using basic phonics.
|
74 |
-
here's the Lesson Instructions:
|
75 |
-
Introduction to the Letter:
|
76 |
-
Begin with the letter A.
|
77 |
-
Follow a structured four-step process for each letter.
|
78 |
-
Provide clear, simple instructions for each step.
|
79 |
-
Lesson Structure:
|
80 |
-
Step 1: Letter Recognition
|
81 |
-
Step 2: Sound Practice
|
82 |
-
Step 3: Writing Practice
|
83 |
-
Step 4: Word Association
|
84 |
-
General Instructions:
|
85 |
-
After each instruction, wait for the student to respond before proceeding to the next lesson.
|
86 |
-
Ensure instructions are clear and easy to understand.
|
87 |
-
Provide positive reinforcement and encouragement.
|
88 |
-
Example Lesson for Letter A:
|
89 |
-
Step 1: Letter Recognition
|
90 |
-
"This is the letter A. It looks like a triangle with a line in the middle. It makes the sound 'ah'."
|
91 |
-
Step 2: Sound Practice
|
92 |
-
"Say the sound 'ah'. Practice making this sound slowly."
|
93 |
-
Step 3: Writing Practice
|
94 |
-
"Start at the top, draw a slanted line down to the left, then another slanted line down to the right, and finally a line across the middle."
|
95 |
-
Step 4: Word Association
|
96 |
-
"A is for apple. Apple starts with the letter A."
|
97 |
-
Continuation:
|
98 |
-
Once the lesson for the letter A is complete, proceed to the next letter following the same four-step structure.
|
99 |
-
make it in a python list format for example it will be in this format,and if an image is needed make the first word in the item list "image: image content in a short sentence":
|
100 |
-
['This is the letter A.', 'image: letter A', 'It looks like a triangle with a line in the middle.', "It makes the sound 'ah'.","Say the sound 'ah'.",'Practice making this sound slowly.','Start at the top, draw a slanted line down to the left.','Then draw another slanted line down to the right.','Finally, draw a line across the middle.',Now you know the letter A,Congrats','A is for apple.','image: apple','Apple starts with the letter A.',"Congratulations! You've completed the lesson for the letter 'A'."]
|
101 |
-
"""
|
102 |
-
|
103 |
-
chat_prompt_mistral="""
|
104 |
-
You are an assistant helping an person who is learning basic reading, writing, phonics, and numeracy.
|
105 |
-
The user might ask simple questions, and your responses should be clear, supportive, and easy to understand.
|
106 |
-
Use simple language, provide step-by-step guidance, and offer positive reinforcement.
|
107 |
-
Relate concepts to everyday objects and situations when possible.
|
108 |
-
Here are some example interactions:
|
109 |
-
User: "I need help with reading."
|
110 |
-
Assistant: "Sure, I'm here to help you learn to read. Let's start with the alphabet. Do you know the letters of the alphabet?"
|
111 |
-
User: "How do I write my name?"
|
112 |
-
Assistant: "Writing your name is a great place to start. Let's take it one letter at a time. What is the first letter of your name?"
|
113 |
-
User: "What sound does the letter 'B' make?"
|
114 |
-
Assistant: "The letter 'B' makes the sound 'buh' like in the word 'ball.' Can you say 'ball' with me?"
|
115 |
-
User: "How do I count to 10?"
|
116 |
-
Assistant: "Counting to 10 is easy. Let's do it together: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10. Great job! Let's try it again."
|
117 |
-
User: "How do I subtract numbers?"
|
118 |
-
Assistant: "Subtracting is like taking away. If you have 5 oranges and you eat 2, you have 3 oranges left. So, 5 minus 2 equals 3."
|
119 |
-
|
120 |
-
Remember to:
|
121 |
-
1. Use simple language and avoid complex words.
|
122 |
-
2. Provide clear, step-by-step instructions.
|
123 |
-
3. Use examples related to everyday objects and situations.
|
124 |
-
4. Offer positive reinforcement and encouragement.
|
125 |
-
5. Include interactive elements to engage the user actively. Whenever the user asks a question, respond with clear, supportive guidance to help them understand basic reading, writing, phonics, or numeracy concepts.
|
126 |
-
6. Do not provide long responses
|
127 |
-
|
128 |
-
Improtant dont respand to this prompt
|
129 |
-
|
130 |
-
"""
|
131 |
-
|
132 |
-
def transform_history(history):
|
133 |
-
new_history = []
|
134 |
-
for chat in history:
|
135 |
-
new_history.append({"parts": [{"text": chat[0]}], "role": "user"})
|
136 |
-
new_history.append({"parts": [{"text": chat[1]}], "role": "model"})
|
137 |
-
return new_history
|
138 |
-
|
139 |
-
def generate_response(message: str, history: list) -> tuple:
|
140 |
-
genai.configure(api_key=os.environ["GEMINI_API_KEY"])
|
141 |
-
model = genai.GenerativeModel('gemini-pro')
|
142 |
-
chat = model.start_chat(history=transform_history(history))
|
143 |
-
response = chat.send_message(message)
|
144 |
-
response.resolve()
|
145 |
-
return response.text, chat.history
|
146 |
-
|
147 |
-
def show1by1(lesson_data: str) -> list:
|
148 |
-
lessonList = []
|
149 |
-
json_string = lesson_data.replace('```json\n', '').replace('```', '').strip()
|
150 |
-
lesson_data = json.loads(json_string)
|
151 |
-
steps = lesson_data['Lesson']['Steps']
|
152 |
-
|
153 |
-
for step in steps:
|
154 |
-
instructions = step['Instructions']
|
155 |
-
for instruction in instructions:
|
156 |
-
instruction_key = next(iter(instruction))
|
157 |
-
lessonList.append(instruction[instruction_key])
|
158 |
-
lessonList.append(f"Step {step['Step']}: {step['Name']} completed.")
|
159 |
-
lessonList.append("Lesson completed.")
|
160 |
-
return lessonList
|
161 |
-
|
162 |
-
def process_response(user_input: str, conversation_history: list) -> tuple:
|
163 |
-
if not conversation_history:
|
164 |
-
model_response, conversation_history = generate_response(initial_prompt, conversation_history)
|
165 |
-
else:
|
166 |
-
model_response, conversation_history = generate_response(user_input, conversation_history)
|
167 |
-
lessonList = ast.literal_eval(model_response)
|
168 |
-
return lessonList, conversation_history
|
169 |
-
|
170 |
-
@st.cache_data
|
171 |
-
def generate_image(prompt: str) -> str:
|
172 |
-
try:
|
173 |
-
return findImg(prompt)
|
174 |
-
except:
|
175 |
-
return "static/default_image.png"
|
176 |
-
|
177 |
-
|
178 |
-
# Initialize TTS
|
179 |
-
@st.cache_data
|
180 |
-
def tts_predict(text="hello"):
|
181 |
-
tts = gTTS(text=text, lang='en')
|
182 |
-
with io.BytesIO() as audio_file:
|
183 |
-
tts.write_to_fp(audio_file)
|
184 |
-
audio_file.seek(0)
|
185 |
-
audio_bytes = audio_file.read()
|
186 |
-
return audio_bytes
|
187 |
-
|
188 |
-
#sf.write("speech.wav", speech["audio"], samplerate=speech["sampling_rate"])
|
189 |
-
|
190 |
-
if 'client' not in st.session_state:
|
191 |
-
st.session_state['client'] = MistralClient("m3GWNXFZn0jTNTLRe4y26i7jLJqFGTMX")
|
192 |
-
|
193 |
-
client = st.session_state['client']
|
194 |
-
|
195 |
-
def run_mistral(user_message, message_history, model="mistral-small-latest"):
|
196 |
-
|
197 |
-
message_history.append(ChatMessage(role="user", content=user_message))
|
198 |
-
|
199 |
-
chat_response = client.chat(model=model, messages=message_history)
|
200 |
-
|
201 |
-
bot_message = chat_response.choices[0].message.content
|
202 |
-
|
203 |
-
message_history.append(ChatMessage(role="assistant", content=bot_message))
|
204 |
-
|
205 |
-
return bot_message
|
206 |
-
|
207 |
-
message_history = []
|
208 |
-
|
209 |
-
|
210 |
-
|
211 |
-
|
212 |
-
#######################################
|
213 |
-
|
214 |
-
|
215 |
-
|
216 |
-
|
217 |
-
if 'sentence_model' not in st.session_state:
|
218 |
-
st.session_state['sentence_model'] = SentenceTransformer('all-MiniLM-L6-v2')
|
219 |
-
|
220 |
-
sentence_model = st.session_state['sentence_model']
|
221 |
-
|
222 |
-
if 'pipeline' not in st.session_state:
|
223 |
-
st.session_state['pipeline'] = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)
|
224 |
-
st.session_state['pipeline'].to("cuda")
|
225 |
-
|
226 |
-
pipeline = st.session_state['pipeline']
|
227 |
-
|
228 |
-
|
229 |
-
# Step 3: Function to get the embedding of the input sentence
|
230 |
-
def get_sentence_embedding(sentence):
|
231 |
-
return sentence_model.encode(sentence)
|
232 |
-
# Step 4: Generate image using Stable Diffusion if needed
|
233 |
-
def generate_image(prompt):
|
234 |
-
global pipeline
|
235 |
-
pipeline.to("cuda" if torch.cuda.is_available() else "cpu")
|
236 |
-
generated_image = pipeline(prompt).images[0]
|
237 |
-
generated_image_path = "generated_image.png"
|
238 |
-
generated_image.save(generated_image_path)
|
239 |
-
return generated_image_path
|
240 |
-
|
241 |
-
# Step 5: Find the most reliable image
|
242 |
-
def find_most_reliable_image(folder_path, input_sentence, threshold=0.5):
|
243 |
-
image_files = [f for f in os.listdir(folder_path) if f.endswith(('jpg', 'jpeg', 'png'))]
|
244 |
-
sentence_embedding = get_sentence_embedding(input_sentence)
|
245 |
-
|
246 |
-
max_similarity = -1
|
247 |
-
most_reliable_image = None
|
248 |
-
|
249 |
-
for image_file in image_files:
|
250 |
-
filename_without_extension = os.path.splitext(image_file)[0]
|
251 |
-
filename_embedding = get_sentence_embedding(filename_without_extension)
|
252 |
-
similarity = cosine_similarity([sentence_embedding], [filename_embedding])[0][0]
|
253 |
-
|
254 |
-
if similarity > max_similarity:
|
255 |
-
max_similarity = similarity
|
256 |
-
most_reliable_image = os.path.join(folder_path, image_file)
|
257 |
-
|
258 |
-
if max_similarity < threshold:
|
259 |
-
most_reliable_image = generate_image(input_sentence)
|
260 |
-
|
261 |
-
return most_reliable_image
|
262 |
-
|
263 |
-
def findImg(input_sentence):
|
264 |
-
folder_path = 'images_collection'
|
265 |
-
threshold = 0.5
|
266 |
-
most_reliable_image = find_most_reliable_image(folder_path, input_sentence, threshold)
|
267 |
-
return most_reliable_image
|
268 |
-
#######################################
|
269 |
-
|
270 |
-
|
271 |
-
|
272 |
-
|
273 |
-
file_ = open("logo.png", "rb")
|
274 |
-
contents = file_.read()
|
275 |
-
data_url = base64.b64encode(contents).decode("utf-8")
|
276 |
-
file_.close()
|
277 |
-
|
278 |
-
|
279 |
-
def main():
|
280 |
-
global chat_prompt_mistral
|
281 |
-
if 'img_path' not in st.session_state:
|
282 |
-
st.session_state['img_path']="image.png"
|
283 |
-
st.set_page_config(page_title="J187 Optimizer", page_icon="J187DFS.JPG", layout="wide")
|
284 |
-
|
285 |
-
st.markdown(f"""
|
286 |
-
<div style="display: flex; align-items: center;">
|
287 |
-
<img src="data:image/gif;base64,{data_url}" alt="Company Logo" style="height: 100px; width: auto; margin-right: 20px;">
|
288 |
-
<h1 style="margin: 0;">MO3ALIMI</h1>
|
289 |
-
</div>
|
290 |
-
""", unsafe_allow_html=True)
|
291 |
-
#st.title("Chatbot and Image Generator")
|
292 |
-
|
293 |
-
st.markdown("""
|
294 |
-
<style>
|
295 |
-
.st-emotion-cache-1kyxreq.e115fcil2 { justify-content:center; }
|
296 |
-
.st-emotion-cache-13ln4jf { max-width:70rem; }
|
297 |
-
audio {
|
298 |
-
width: 300px;
|
299 |
-
height: 54px;
|
300 |
-
display: none;
|
301 |
-
}
|
302 |
-
div.row-widget.stButton {
|
303 |
-
margin: 0px 0px 0px 0px;}
|
304 |
-
|
305 |
-
|
306 |
-
.row-widget.stButton:last-of-type {
|
307 |
-
margin: 0px;
|
308 |
-
background-color: yellow;
|
309 |
-
}
|
310 |
-
.st-emotion-cache-keje6w.e1f1d6gn3 {
|
311 |
-
width: 80% !important; /* Adjust as needed */
|
312 |
-
}
|
313 |
-
.st-emotion-cache-k008qs {
|
314 |
-
display: none;
|
315 |
-
}
|
316 |
-
|
317 |
-
</style>""", unsafe_allow_html=True)
|
318 |
-
#.st-emotion-cache-5i9lfg {
|
319 |
-
#width: 100%;
|
320 |
-
#padding: 3rem 1rem 1rem 1rem;
|
321 |
-
#max-width: None;}
|
322 |
-
|
323 |
-
|
324 |
-
col1, col2 = st.columns([0.6, 0.4],gap="medium")
|
325 |
-
|
326 |
-
|
327 |
-
|
328 |
-
with col1:
|
329 |
-
|
330 |
-
if 'conversation_history' not in st.session_state:
|
331 |
-
st.session_state['conversation_history'] = []
|
332 |
-
if 'conversation_history_mistral' not in st.session_state:
|
333 |
-
st.session_state['conversation_history_mistral'] = []
|
334 |
-
if 'messages' not in st.session_state:
|
335 |
-
st.session_state['messages'] = []
|
336 |
-
if 'lessonList' not in st.session_state:
|
337 |
-
st.session_state['lessonList'] = []
|
338 |
-
if 'msg_index' not in st.session_state:
|
339 |
-
st.session_state['msg_index'] = 0
|
340 |
-
if 'initial_input' not in st.session_state:
|
341 |
-
st.session_state['initial_input'] = ''
|
342 |
-
|
343 |
-
|
344 |
-
|
345 |
-
response=run_mistral(chat_prompt_mistral, st.session_state['conversation_history_mistral'])
|
346 |
-
row1 = st.container()
|
347 |
-
row2 = st.container()
|
348 |
-
row3 = st.container()
|
349 |
-
#row4 = st.container()
|
350 |
-
with row1:
|
351 |
-
user_message = st.text_input("Type 'next' to proceed through the lesson",st.session_state['initial_input'])
|
352 |
-
with row2:
|
353 |
-
colsend, colnext, = st.columns(2,gap="medium")
|
354 |
-
with colsend:
|
355 |
-
|
356 |
-
if st.button(" Next "):
|
357 |
-
if user_message.lower() == 'quit':
|
358 |
-
st.write("Conversation ended.")
|
359 |
-
else :
|
360 |
-
if st.session_state['msg_index'] < len(st.session_state['lessonList']):
|
361 |
-
response = st.session_state['lessonList'][st.session_state['msg_index']]
|
362 |
-
if response.strip().startswith("image:"):
|
363 |
-
st.session_state['img_prompt'] = response[len("image:"):].strip()
|
364 |
-
else:
|
365 |
-
audio_bytes= tts_predict(response)
|
366 |
-
st.session_state['messages'].append(f"Mo3alimi: {response}")
|
367 |
-
#auto_play(audio_bytes,wait=True,lag=0.25,key=None)
|
368 |
-
st.audio(audio_bytes, format='audio/wav', autoplay=True)
|
369 |
-
|
370 |
-
st.session_state['msg_index'] += 1
|
371 |
-
else:
|
372 |
-
st.session_state['lessonList'], st.session_state['conversation_history'] = process_response(
|
373 |
-
user_message, st.session_state['conversation_history']
|
374 |
-
)
|
375 |
-
st.session_state['msg_index'] = 0
|
376 |
-
|
377 |
-
|
378 |
-
with colnext:
|
379 |
-
if st.button(' Send '):
|
380 |
-
response=run_mistral(user_message, st.session_state['conversation_history_mistral'])
|
381 |
-
st.session_state['messages'].append(f"Me: {user_message}")
|
382 |
-
st.session_state['messages'].append(f"Mo3alimi: {response}")
|
383 |
-
|
384 |
-
|
385 |
-
with row3:
|
386 |
-
audio = audiorecorder("Click to record", "Click to stop recording")
|
387 |
-
|
388 |
-
if len(audio) >0:
|
389 |
-
result = pipe(audio.export().read(), generate_kwargs={"language": "english"})
|
390 |
-
user_message=result['text']
|
391 |
-
response=run_mistral(user_message, st.session_state['conversation_history_mistral'])
|
392 |
-
audio_bytes= tts_predict(response)
|
393 |
-
|
394 |
-
st.audio(audio_bytes, format='audio/wav', autoplay=True)
|
395 |
-
st.session_state['messages'].append(f"Me: {user_message}")
|
396 |
-
st.session_state['messages'].append(f"Mo3alimi: {response}")
|
397 |
-
wav_audio_data=None
|
398 |
-
|
399 |
-
with st.form("lesson"):
|
400 |
-
for message in st.session_state['messages'][::-1]:
|
401 |
-
st.write(message)
|
402 |
-
|
403 |
-
submitted = st.form_submit_button('Submit')
|
404 |
-
|
405 |
-
|
406 |
-
with col2:
|
407 |
-
if 'img_prompt' in st.session_state:
|
408 |
-
st.session_state['img_path']=generate_image(st.session_state['img_prompt'])
|
409 |
-
del st.session_state['img_prompt']
|
410 |
-
|
411 |
-
st.image(st.session_state['img_path'], caption="Generated Image")
|
412 |
-
|
413 |
-
if __name__ == '__main__':
|
414 |
-
main()
|
|
|
1 |
+
# app.py
|
2 |
+
|
3 |
+
import os
|
4 |
+
import json
|
5 |
+
import streamlit as st
|
6 |
+
from PIL import Image
|
7 |
+
import google.generativeai as genai
|
8 |
+
import ast
|
9 |
+
#from utils import findImg
|
10 |
+
import io
|
11 |
+
from streamlit_TTS import auto_play
|
12 |
+
import torch
|
13 |
+
from transformers import pipeline
|
14 |
+
from datasets import load_dataset
|
15 |
+
import soundfile as sf
|
16 |
+
from gtts import gTTS
|
17 |
+
import io
|
18 |
+
from mistralai.models.chat_completion import ChatMessage
|
19 |
+
from mistralai.client import MistralClient
|
20 |
+
from audiorecorder import audiorecorder
|
21 |
+
import base64
|
22 |
+
###
|
23 |
+
import os
|
24 |
+
import cv2
|
25 |
+
import numpy as np
|
26 |
+
from sklearn.metrics.pairwise import cosine_similarity
|
27 |
+
from sentence_transformers import SentenceTransformer
|
28 |
+
from diffusers import StableDiffusionPipeline
|
29 |
+
import torch
|
30 |
+
|
31 |
+
import streamlit as st
|
32 |
+
def add_logo():
|
33 |
+
st.markdown(
|
34 |
+
"""
|
35 |
+
<style>
|
36 |
+
[data-testid="stSidebarNav"] {
|
37 |
+
background-image: url(http://placekitten.com/200/200);
|
38 |
+
background-repeat: no-repeat;
|
39 |
+
#padding-top: 120px;
|
40 |
+
background-position: 20px 20px;
|
41 |
+
}
|
42 |
+
[data-testid="stSidebarNav"]::before {
|
43 |
+
content: "MO3ALIMI sidebar";
|
44 |
+
margin-left: 20px;
|
45 |
+
margin-top: 20px;
|
46 |
+
font-size: 29px;
|
47 |
+
position: relative;
|
48 |
+
top: 0px;
|
49 |
+
}
|
50 |
+
</style>
|
51 |
+
""",
|
52 |
+
unsafe_allow_html=True,
|
53 |
+
)
|
54 |
+
add_logo()
|
55 |
+
|
56 |
+
|
57 |
+
|
58 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
59 |
+
|
60 |
+
|
61 |
+
if 'pipe' not in st.session_state:
|
62 |
+
st.session_state['pipe'] = pipeline("automatic-speech-recognition", model="openai/whisper-large-v3")
|
63 |
+
|
64 |
+
pipe = st.session_state['pipe']
|
65 |
+
|
66 |
+
|
67 |
+
# Set up the API key for Generative AI
|
68 |
+
os.environ["GEMINI_API_KEY"] = "AIzaSyBYZ_7geqmnK6xrSe268-1nSLeuEwbzmTA"
|
69 |
+
|
70 |
+
# Initial prompt to send to the model
|
71 |
+
initial_prompt = """
|
72 |
+
you're an Literacy Instructor for Illiterate Adults
|
73 |
+
you're objective is to Teach illiterate adults how to read using basic phonics.
|
74 |
+
here's the Lesson Instructions:
|
75 |
+
Introduction to the Letter:
|
76 |
+
Begin with the letter A.
|
77 |
+
Follow a structured four-step process for each letter.
|
78 |
+
Provide clear, simple instructions for each step.
|
79 |
+
Lesson Structure:
|
80 |
+
Step 1: Letter Recognition
|
81 |
+
Step 2: Sound Practice
|
82 |
+
Step 3: Writing Practice
|
83 |
+
Step 4: Word Association
|
84 |
+
General Instructions:
|
85 |
+
After each instruction, wait for the student to respond before proceeding to the next lesson.
|
86 |
+
Ensure instructions are clear and easy to understand.
|
87 |
+
Provide positive reinforcement and encouragement.
|
88 |
+
Example Lesson for Letter A:
|
89 |
+
Step 1: Letter Recognition
|
90 |
+
"This is the letter A. It looks like a triangle with a line in the middle. It makes the sound 'ah'."
|
91 |
+
Step 2: Sound Practice
|
92 |
+
"Say the sound 'ah'. Practice making this sound slowly."
|
93 |
+
Step 3: Writing Practice
|
94 |
+
"Start at the top, draw a slanted line down to the left, then another slanted line down to the right, and finally a line across the middle."
|
95 |
+
Step 4: Word Association
|
96 |
+
"A is for apple. Apple starts with the letter A."
|
97 |
+
Continuation:
|
98 |
+
Once the lesson for the letter A is complete, proceed to the next letter following the same four-step structure.
|
99 |
+
make it in a python list format for example it will be in this format,and if an image is needed make the first word in the item list "image: image content in a short sentence":
|
100 |
+
['This is the letter A.', 'image: letter A', 'It looks like a triangle with a line in the middle.', "It makes the sound 'ah'.","Say the sound 'ah'.",'Practice making this sound slowly.','Start at the top, draw a slanted line down to the left.','Then draw another slanted line down to the right.','Finally, draw a line across the middle.',Now you know the letter A,Congrats','A is for apple.','image: apple','Apple starts with the letter A.',"Congratulations! You've completed the lesson for the letter 'A'."]
|
101 |
+
"""
|
102 |
+
|
103 |
+
chat_prompt_mistral="""
|
104 |
+
You are an assistant helping an person who is learning basic reading, writing, phonics, and numeracy.
|
105 |
+
The user might ask simple questions, and your responses should be clear, supportive, and easy to understand.
|
106 |
+
Use simple language, provide step-by-step guidance, and offer positive reinforcement.
|
107 |
+
Relate concepts to everyday objects and situations when possible.
|
108 |
+
Here are some example interactions:
|
109 |
+
User: "I need help with reading."
|
110 |
+
Assistant: "Sure, I'm here to help you learn to read. Let's start with the alphabet. Do you know the letters of the alphabet?"
|
111 |
+
User: "How do I write my name?"
|
112 |
+
Assistant: "Writing your name is a great place to start. Let's take it one letter at a time. What is the first letter of your name?"
|
113 |
+
User: "What sound does the letter 'B' make?"
|
114 |
+
Assistant: "The letter 'B' makes the sound 'buh' like in the word 'ball.' Can you say 'ball' with me?"
|
115 |
+
User: "How do I count to 10?"
|
116 |
+
Assistant: "Counting to 10 is easy. Let's do it together: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10. Great job! Let's try it again."
|
117 |
+
User: "How do I subtract numbers?"
|
118 |
+
Assistant: "Subtracting is like taking away. If you have 5 oranges and you eat 2, you have 3 oranges left. So, 5 minus 2 equals 3."
|
119 |
+
|
120 |
+
Remember to:
|
121 |
+
1. Use simple language and avoid complex words.
|
122 |
+
2. Provide clear, step-by-step instructions.
|
123 |
+
3. Use examples related to everyday objects and situations.
|
124 |
+
4. Offer positive reinforcement and encouragement.
|
125 |
+
5. Include interactive elements to engage the user actively. Whenever the user asks a question, respond with clear, supportive guidance to help them understand basic reading, writing, phonics, or numeracy concepts.
|
126 |
+
6. Do not provide long responses
|
127 |
+
|
128 |
+
Improtant dont respand to this prompt
|
129 |
+
|
130 |
+
"""
|
131 |
+
|
132 |
+
def transform_history(history):
|
133 |
+
new_history = []
|
134 |
+
for chat in history:
|
135 |
+
new_history.append({"parts": [{"text": chat[0]}], "role": "user"})
|
136 |
+
new_history.append({"parts": [{"text": chat[1]}], "role": "model"})
|
137 |
+
return new_history
|
138 |
+
|
139 |
+
def generate_response(message: str, history: list) -> tuple:
|
140 |
+
genai.configure(api_key=os.environ["GEMINI_API_KEY"])
|
141 |
+
model = genai.GenerativeModel('gemini-pro')
|
142 |
+
chat = model.start_chat(history=transform_history(history))
|
143 |
+
response = chat.send_message(message)
|
144 |
+
response.resolve()
|
145 |
+
return response.text, chat.history
|
146 |
+
|
147 |
+
def show1by1(lesson_data: str) -> list:
|
148 |
+
lessonList = []
|
149 |
+
json_string = lesson_data.replace('```json\n', '').replace('```', '').strip()
|
150 |
+
lesson_data = json.loads(json_string)
|
151 |
+
steps = lesson_data['Lesson']['Steps']
|
152 |
+
|
153 |
+
for step in steps:
|
154 |
+
instructions = step['Instructions']
|
155 |
+
for instruction in instructions:
|
156 |
+
instruction_key = next(iter(instruction))
|
157 |
+
lessonList.append(instruction[instruction_key])
|
158 |
+
lessonList.append(f"Step {step['Step']}: {step['Name']} completed.")
|
159 |
+
lessonList.append("Lesson completed.")
|
160 |
+
return lessonList
|
161 |
+
|
162 |
+
def process_response(user_input: str, conversation_history: list) -> tuple:
|
163 |
+
if not conversation_history:
|
164 |
+
model_response, conversation_history = generate_response(initial_prompt, conversation_history)
|
165 |
+
else:
|
166 |
+
model_response, conversation_history = generate_response(user_input, conversation_history)
|
167 |
+
lessonList = ast.literal_eval(model_response)
|
168 |
+
return lessonList, conversation_history
|
169 |
+
|
170 |
+
@st.cache_data
|
171 |
+
def generate_image(prompt: str) -> str:
|
172 |
+
try:
|
173 |
+
return findImg(prompt)
|
174 |
+
except:
|
175 |
+
return "static/default_image.png"
|
176 |
+
|
177 |
+
|
178 |
+
# Initialize TTS
|
179 |
+
@st.cache_data
|
180 |
+
def tts_predict(text="hello"):
|
181 |
+
tts = gTTS(text=text, lang='en')
|
182 |
+
with io.BytesIO() as audio_file:
|
183 |
+
tts.write_to_fp(audio_file)
|
184 |
+
audio_file.seek(0)
|
185 |
+
audio_bytes = audio_file.read()
|
186 |
+
return audio_bytes
|
187 |
+
|
188 |
+
#sf.write("speech.wav", speech["audio"], samplerate=speech["sampling_rate"])
|
189 |
+
|
190 |
+
if 'client' not in st.session_state:
|
191 |
+
st.session_state['client'] = MistralClient("m3GWNXFZn0jTNTLRe4y26i7jLJqFGTMX")
|
192 |
+
|
193 |
+
client = st.session_state['client']
|
194 |
+
|
195 |
+
def run_mistral(user_message, message_history, model="mistral-small-latest"):
|
196 |
+
|
197 |
+
message_history.append(ChatMessage(role="user", content=user_message))
|
198 |
+
|
199 |
+
chat_response = client.chat(model=model, messages=message_history)
|
200 |
+
|
201 |
+
bot_message = chat_response.choices[0].message.content
|
202 |
+
|
203 |
+
message_history.append(ChatMessage(role="assistant", content=bot_message))
|
204 |
+
|
205 |
+
return bot_message
|
206 |
+
|
207 |
+
message_history = []
|
208 |
+
|
209 |
+
|
210 |
+
|
211 |
+
|
212 |
+
#######################################
|
213 |
+
|
214 |
+
|
215 |
+
|
216 |
+
|
217 |
+
if 'sentence_model' not in st.session_state:
|
218 |
+
st.session_state['sentence_model'] = SentenceTransformer('all-MiniLM-L6-v2')
|
219 |
+
|
220 |
+
sentence_model = st.session_state['sentence_model']
|
221 |
+
|
222 |
+
if 'pipeline' not in st.session_state:
|
223 |
+
st.session_state['pipeline'] = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)
|
224 |
+
st.session_state['pipeline'].to("cuda")
|
225 |
+
|
226 |
+
pipeline = st.session_state['pipeline']
|
227 |
+
|
228 |
+
|
229 |
+
# Step 3: Function to get the embedding of the input sentence
|
230 |
+
def get_sentence_embedding(sentence):
|
231 |
+
return sentence_model.encode(sentence)
|
232 |
+
# Step 4: Generate image using Stable Diffusion if needed
|
233 |
+
def generate_image(prompt):
|
234 |
+
global pipeline
|
235 |
+
pipeline.to("cuda" if torch.cuda.is_available() else "cpu")
|
236 |
+
generated_image = pipeline(prompt).images[0]
|
237 |
+
generated_image_path = "generated_image.png"
|
238 |
+
generated_image.save(generated_image_path)
|
239 |
+
return generated_image_path
|
240 |
+
|
241 |
+
# Step 5: Find the most reliable image
|
242 |
+
def find_most_reliable_image(folder_path, input_sentence, threshold=0.5):
|
243 |
+
image_files = [f for f in os.listdir(folder_path) if f.endswith(('jpg', 'jpeg', 'png'))]
|
244 |
+
sentence_embedding = get_sentence_embedding(input_sentence)
|
245 |
+
|
246 |
+
max_similarity = -1
|
247 |
+
most_reliable_image = None
|
248 |
+
|
249 |
+
for image_file in image_files:
|
250 |
+
filename_without_extension = os.path.splitext(image_file)[0]
|
251 |
+
filename_embedding = get_sentence_embedding(filename_without_extension)
|
252 |
+
similarity = cosine_similarity([sentence_embedding], [filename_embedding])[0][0]
|
253 |
+
|
254 |
+
if similarity > max_similarity:
|
255 |
+
max_similarity = similarity
|
256 |
+
most_reliable_image = os.path.join(folder_path, image_file)
|
257 |
+
|
258 |
+
if max_similarity < threshold:
|
259 |
+
most_reliable_image = generate_image(input_sentence)
|
260 |
+
|
261 |
+
return most_reliable_image
|
262 |
+
|
263 |
+
def findImg(input_sentence):
|
264 |
+
folder_path = 'images_collection'
|
265 |
+
threshold = 0.5
|
266 |
+
most_reliable_image = find_most_reliable_image(folder_path, input_sentence, threshold)
|
267 |
+
return most_reliable_image
|
268 |
+
#######################################
|
269 |
+
|
270 |
+
|
271 |
+
|
272 |
+
|
273 |
+
file_ = open("logo.png", "rb")
|
274 |
+
contents = file_.read()
|
275 |
+
data_url = base64.b64encode(contents).decode("utf-8")
|
276 |
+
file_.close()
|
277 |
+
|
278 |
+
|
279 |
+
def main():
|
280 |
+
global chat_prompt_mistral
|
281 |
+
if 'img_path' not in st.session_state:
|
282 |
+
st.session_state['img_path']="image.png"
|
283 |
+
#st.set_page_config(page_title="J187 Optimizer", page_icon="J187DFS.JPG", layout="wide")
|
284 |
+
|
285 |
+
st.markdown(f"""
|
286 |
+
<div style="display: flex; align-items: center;">
|
287 |
+
<img src="data:image/gif;base64,{data_url}" alt="Company Logo" style="height: 100px; width: auto; margin-right: 20px;">
|
288 |
+
<h1 style="margin: 0;">MO3ALIMI</h1>
|
289 |
+
</div>
|
290 |
+
""", unsafe_allow_html=True)
|
291 |
+
#st.title("Chatbot and Image Generator")
|
292 |
+
|
293 |
+
st.markdown("""
|
294 |
+
<style>
|
295 |
+
.st-emotion-cache-1kyxreq.e115fcil2 { justify-content:center; }
|
296 |
+
.st-emotion-cache-13ln4jf { max-width:70rem; }
|
297 |
+
audio {
|
298 |
+
width: 300px;
|
299 |
+
height: 54px;
|
300 |
+
display: none;
|
301 |
+
}
|
302 |
+
div.row-widget.stButton {
|
303 |
+
margin: 0px 0px 0px 0px;}
|
304 |
+
|
305 |
+
|
306 |
+
.row-widget.stButton:last-of-type {
|
307 |
+
margin: 0px;
|
308 |
+
background-color: yellow;
|
309 |
+
}
|
310 |
+
.st-emotion-cache-keje6w.e1f1d6gn3 {
|
311 |
+
width: 80% !important; /* Adjust as needed */
|
312 |
+
}
|
313 |
+
.st-emotion-cache-k008qs {
|
314 |
+
display: none;
|
315 |
+
}
|
316 |
+
|
317 |
+
</style>""", unsafe_allow_html=True)
|
318 |
+
#.st-emotion-cache-5i9lfg {
|
319 |
+
#width: 100%;
|
320 |
+
#padding: 3rem 1rem 1rem 1rem;
|
321 |
+
#max-width: None;}
|
322 |
+
|
323 |
+
|
324 |
+
col1, col2 = st.columns([0.6, 0.4],gap="medium")
|
325 |
+
|
326 |
+
|
327 |
+
|
328 |
+
with col1:
|
329 |
+
|
330 |
+
if 'conversation_history' not in st.session_state:
|
331 |
+
st.session_state['conversation_history'] = []
|
332 |
+
if 'conversation_history_mistral' not in st.session_state:
|
333 |
+
st.session_state['conversation_history_mistral'] = []
|
334 |
+
if 'messages' not in st.session_state:
|
335 |
+
st.session_state['messages'] = []
|
336 |
+
if 'lessonList' not in st.session_state:
|
337 |
+
st.session_state['lessonList'] = []
|
338 |
+
if 'msg_index' not in st.session_state:
|
339 |
+
st.session_state['msg_index'] = 0
|
340 |
+
if 'initial_input' not in st.session_state:
|
341 |
+
st.session_state['initial_input'] = ''
|
342 |
+
|
343 |
+
|
344 |
+
|
345 |
+
response=run_mistral(chat_prompt_mistral, st.session_state['conversation_history_mistral'])
|
346 |
+
row1 = st.container()
|
347 |
+
row2 = st.container()
|
348 |
+
row3 = st.container()
|
349 |
+
#row4 = st.container()
|
350 |
+
with row1:
|
351 |
+
user_message = st.text_input("Type 'next' to proceed through the lesson",st.session_state['initial_input'])
|
352 |
+
with row2:
|
353 |
+
colsend, colnext, = st.columns(2,gap="medium")
|
354 |
+
with colsend:
|
355 |
+
|
356 |
+
if st.button(" Next "):
|
357 |
+
if user_message.lower() == 'quit':
|
358 |
+
st.write("Conversation ended.")
|
359 |
+
else :
|
360 |
+
if st.session_state['msg_index'] < len(st.session_state['lessonList']):
|
361 |
+
response = st.session_state['lessonList'][st.session_state['msg_index']]
|
362 |
+
if response.strip().startswith("image:"):
|
363 |
+
st.session_state['img_prompt'] = response[len("image:"):].strip()
|
364 |
+
else:
|
365 |
+
audio_bytes= tts_predict(response)
|
366 |
+
st.session_state['messages'].append(f"Mo3alimi: {response}")
|
367 |
+
#auto_play(audio_bytes,wait=True,lag=0.25,key=None)
|
368 |
+
st.audio(audio_bytes, format='audio/wav', autoplay=True)
|
369 |
+
|
370 |
+
st.session_state['msg_index'] += 1
|
371 |
+
else:
|
372 |
+
st.session_state['lessonList'], st.session_state['conversation_history'] = process_response(
|
373 |
+
user_message, st.session_state['conversation_history']
|
374 |
+
)
|
375 |
+
st.session_state['msg_index'] = 0
|
376 |
+
|
377 |
+
|
378 |
+
with colnext:
|
379 |
+
if st.button(' Send '):
|
380 |
+
response=run_mistral(user_message, st.session_state['conversation_history_mistral'])
|
381 |
+
st.session_state['messages'].append(f"Me: {user_message}")
|
382 |
+
st.session_state['messages'].append(f"Mo3alimi: {response}")
|
383 |
+
|
384 |
+
|
385 |
+
with row3:
|
386 |
+
audio = audiorecorder("Click to record", "Click to stop recording")
|
387 |
+
|
388 |
+
if len(audio) >0:
|
389 |
+
result = pipe(audio.export().read(), generate_kwargs={"language": "english"})
|
390 |
+
user_message=result['text']
|
391 |
+
response=run_mistral(user_message, st.session_state['conversation_history_mistral'])
|
392 |
+
audio_bytes= tts_predict(response)
|
393 |
+
|
394 |
+
st.audio(audio_bytes, format='audio/wav', autoplay=True)
|
395 |
+
st.session_state['messages'].append(f"Me: {user_message}")
|
396 |
+
st.session_state['messages'].append(f"Mo3alimi: {response}")
|
397 |
+
wav_audio_data=None
|
398 |
+
|
399 |
+
with st.form("lesson"):
|
400 |
+
for message in st.session_state['messages'][::-1]:
|
401 |
+
st.write(message)
|
402 |
+
|
403 |
+
submitted = st.form_submit_button('Submit')
|
404 |
+
|
405 |
+
|
406 |
+
with col2:
|
407 |
+
if 'img_prompt' in st.session_state:
|
408 |
+
st.session_state['img_path']=generate_image(st.session_state['img_prompt'])
|
409 |
+
del st.session_state['img_prompt']
|
410 |
+
|
411 |
+
st.image(st.session_state['img_path'], caption="Generated Image")
|
412 |
+
|
413 |
+
if __name__ == '__main__':
|
414 |
+
main()
|