mouadenna commited on
Commit
2853e45
·
verified ·
1 Parent(s): 7f383cf

Update pages/2_phonics.py

Browse files
Files changed (1) hide show
  1. pages/2_phonics.py +414 -414
pages/2_phonics.py CHANGED
@@ -1,414 +1,414 @@
1
- # app.py
2
-
3
- import os
4
- import json
5
- import streamlit as st
6
- from PIL import Image
7
- import google.generativeai as genai
8
- import ast
9
- #from utils import findImg
10
- import io
11
- from streamlit_TTS import auto_play
12
- import torch
13
- from transformers import pipeline
14
- from datasets import load_dataset
15
- import soundfile as sf
16
- from gtts import gTTS
17
- import io
18
- from mistralai.models.chat_completion import ChatMessage
19
- from mistralai.client import MistralClient
20
- from audiorecorder import audiorecorder
21
- import base64
22
- ###
23
- import os
24
- import cv2
25
- import numpy as np
26
- from sklearn.metrics.pairwise import cosine_similarity
27
- from sentence_transformers import SentenceTransformer
28
- from diffusers import StableDiffusionPipeline
29
- import torch
30
-
31
- import streamlit as st
32
- def add_logo():
33
- st.markdown(
34
- """
35
- <style>
36
- [data-testid="stSidebarNav"] {
37
- background-image: url(http://placekitten.com/200/200);
38
- background-repeat: no-repeat;
39
- #padding-top: 120px;
40
- background-position: 20px 20px;
41
- }
42
- [data-testid="stSidebarNav"]::before {
43
- content: "MO3ALIMI sidebar";
44
- margin-left: 20px;
45
- margin-top: 20px;
46
- font-size: 29px;
47
- position: relative;
48
- top: 0px;
49
- }
50
- </style>
51
- """,
52
- unsafe_allow_html=True,
53
- )
54
- add_logo()
55
-
56
-
57
-
58
- device = "cuda" if torch.cuda.is_available() else "cpu"
59
-
60
-
61
- if 'pipe' not in st.session_state:
62
- st.session_state['pipe'] = pipeline("automatic-speech-recognition", model="openai/whisper-large-v3")
63
-
64
- pipe = st.session_state['pipe']
65
-
66
-
67
- # Set up the API key for Generative AI
68
- os.environ["GEMINI_API_KEY"] = "AIzaSyBYZ_7geqmnK6xrSe268-1nSLeuEwbzmTA"
69
-
70
- # Initial prompt to send to the model
71
- initial_prompt = """
72
- you're an Literacy Instructor for Illiterate Adults
73
- you're objective is to Teach illiterate adults how to read using basic phonics.
74
- here's the Lesson Instructions:
75
- Introduction to the Letter:
76
- Begin with the letter A.
77
- Follow a structured four-step process for each letter.
78
- Provide clear, simple instructions for each step.
79
- Lesson Structure:
80
- Step 1: Letter Recognition
81
- Step 2: Sound Practice
82
- Step 3: Writing Practice
83
- Step 4: Word Association
84
- General Instructions:
85
- After each instruction, wait for the student to respond before proceeding to the next lesson.
86
- Ensure instructions are clear and easy to understand.
87
- Provide positive reinforcement and encouragement.
88
- Example Lesson for Letter A:
89
- Step 1: Letter Recognition
90
- "This is the letter A. It looks like a triangle with a line in the middle. It makes the sound 'ah'."
91
- Step 2: Sound Practice
92
- "Say the sound 'ah'. Practice making this sound slowly."
93
- Step 3: Writing Practice
94
- "Start at the top, draw a slanted line down to the left, then another slanted line down to the right, and finally a line across the middle."
95
- Step 4: Word Association
96
- "A is for apple. Apple starts with the letter A."
97
- Continuation:
98
- Once the lesson for the letter A is complete, proceed to the next letter following the same four-step structure.
99
- make it in a python list format for example it will be in this format,and if an image is needed make the first word in the item list "image: image content in a short sentence":
100
- ['This is the letter A.', 'image: letter A', 'It looks like a triangle with a line in the middle.', "It makes the sound 'ah'.","Say the sound 'ah'.",'Practice making this sound slowly.','Start at the top, draw a slanted line down to the left.','Then draw another slanted line down to the right.','Finally, draw a line across the middle.',Now you know the letter A,Congrats','A is for apple.','image: apple','Apple starts with the letter A.',"Congratulations! You've completed the lesson for the letter 'A'."]
101
- """
102
-
103
- chat_prompt_mistral="""
104
- You are an assistant helping an person who is learning basic reading, writing, phonics, and numeracy.
105
- The user might ask simple questions, and your responses should be clear, supportive, and easy to understand.
106
- Use simple language, provide step-by-step guidance, and offer positive reinforcement.
107
- Relate concepts to everyday objects and situations when possible.
108
- Here are some example interactions:
109
- User: "I need help with reading."
110
- Assistant: "Sure, I'm here to help you learn to read. Let's start with the alphabet. Do you know the letters of the alphabet?"
111
- User: "How do I write my name?"
112
- Assistant: "Writing your name is a great place to start. Let's take it one letter at a time. What is the first letter of your name?"
113
- User: "What sound does the letter 'B' make?"
114
- Assistant: "The letter 'B' makes the sound 'buh' like in the word 'ball.' Can you say 'ball' with me?"
115
- User: "How do I count to 10?"
116
- Assistant: "Counting to 10 is easy. Let's do it together: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10. Great job! Let's try it again."
117
- User: "How do I subtract numbers?"
118
- Assistant: "Subtracting is like taking away. If you have 5 oranges and you eat 2, you have 3 oranges left. So, 5 minus 2 equals 3."
119
-
120
- Remember to:
121
- 1. Use simple language and avoid complex words.
122
- 2. Provide clear, step-by-step instructions.
123
- 3. Use examples related to everyday objects and situations.
124
- 4. Offer positive reinforcement and encouragement.
125
- 5. Include interactive elements to engage the user actively. Whenever the user asks a question, respond with clear, supportive guidance to help them understand basic reading, writing, phonics, or numeracy concepts.
126
- 6. Do not provide long responses
127
-
128
- Improtant dont respand to this prompt
129
-
130
- """
131
-
132
- def transform_history(history):
133
- new_history = []
134
- for chat in history:
135
- new_history.append({"parts": [{"text": chat[0]}], "role": "user"})
136
- new_history.append({"parts": [{"text": chat[1]}], "role": "model"})
137
- return new_history
138
-
139
- def generate_response(message: str, history: list) -> tuple:
140
- genai.configure(api_key=os.environ["GEMINI_API_KEY"])
141
- model = genai.GenerativeModel('gemini-pro')
142
- chat = model.start_chat(history=transform_history(history))
143
- response = chat.send_message(message)
144
- response.resolve()
145
- return response.text, chat.history
146
-
147
- def show1by1(lesson_data: str) -> list:
148
- lessonList = []
149
- json_string = lesson_data.replace('```json\n', '').replace('```', '').strip()
150
- lesson_data = json.loads(json_string)
151
- steps = lesson_data['Lesson']['Steps']
152
-
153
- for step in steps:
154
- instructions = step['Instructions']
155
- for instruction in instructions:
156
- instruction_key = next(iter(instruction))
157
- lessonList.append(instruction[instruction_key])
158
- lessonList.append(f"Step {step['Step']}: {step['Name']} completed.")
159
- lessonList.append("Lesson completed.")
160
- return lessonList
161
-
162
- def process_response(user_input: str, conversation_history: list) -> tuple:
163
- if not conversation_history:
164
- model_response, conversation_history = generate_response(initial_prompt, conversation_history)
165
- else:
166
- model_response, conversation_history = generate_response(user_input, conversation_history)
167
- lessonList = ast.literal_eval(model_response)
168
- return lessonList, conversation_history
169
-
170
- @st.cache_data
171
- def generate_image(prompt: str) -> str:
172
- try:
173
- return findImg(prompt)
174
- except:
175
- return "static/default_image.png"
176
-
177
-
178
- # Initialize TTS
179
- @st.cache_data
180
- def tts_predict(text="hello"):
181
- tts = gTTS(text=text, lang='en')
182
- with io.BytesIO() as audio_file:
183
- tts.write_to_fp(audio_file)
184
- audio_file.seek(0)
185
- audio_bytes = audio_file.read()
186
- return audio_bytes
187
-
188
- #sf.write("speech.wav", speech["audio"], samplerate=speech["sampling_rate"])
189
-
190
- if 'client' not in st.session_state:
191
- st.session_state['client'] = MistralClient("m3GWNXFZn0jTNTLRe4y26i7jLJqFGTMX")
192
-
193
- client = st.session_state['client']
194
-
195
- def run_mistral(user_message, message_history, model="mistral-small-latest"):
196
-
197
- message_history.append(ChatMessage(role="user", content=user_message))
198
-
199
- chat_response = client.chat(model=model, messages=message_history)
200
-
201
- bot_message = chat_response.choices[0].message.content
202
-
203
- message_history.append(ChatMessage(role="assistant", content=bot_message))
204
-
205
- return bot_message
206
-
207
- message_history = []
208
-
209
-
210
-
211
-
212
- #######################################
213
-
214
-
215
-
216
-
217
- if 'sentence_model' not in st.session_state:
218
- st.session_state['sentence_model'] = SentenceTransformer('all-MiniLM-L6-v2')
219
-
220
- sentence_model = st.session_state['sentence_model']
221
-
222
- if 'pipeline' not in st.session_state:
223
- st.session_state['pipeline'] = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)
224
- st.session_state['pipeline'].to("cuda")
225
-
226
- pipeline = st.session_state['pipeline']
227
-
228
-
229
- # Step 3: Function to get the embedding of the input sentence
230
- def get_sentence_embedding(sentence):
231
- return sentence_model.encode(sentence)
232
- # Step 4: Generate image using Stable Diffusion if needed
233
- def generate_image(prompt):
234
- global pipeline
235
- pipeline.to("cuda" if torch.cuda.is_available() else "cpu")
236
- generated_image = pipeline(prompt).images[0]
237
- generated_image_path = "generated_image.png"
238
- generated_image.save(generated_image_path)
239
- return generated_image_path
240
-
241
- # Step 5: Find the most reliable image
242
- def find_most_reliable_image(folder_path, input_sentence, threshold=0.5):
243
- image_files = [f for f in os.listdir(folder_path) if f.endswith(('jpg', 'jpeg', 'png'))]
244
- sentence_embedding = get_sentence_embedding(input_sentence)
245
-
246
- max_similarity = -1
247
- most_reliable_image = None
248
-
249
- for image_file in image_files:
250
- filename_without_extension = os.path.splitext(image_file)[0]
251
- filename_embedding = get_sentence_embedding(filename_without_extension)
252
- similarity = cosine_similarity([sentence_embedding], [filename_embedding])[0][0]
253
-
254
- if similarity > max_similarity:
255
- max_similarity = similarity
256
- most_reliable_image = os.path.join(folder_path, image_file)
257
-
258
- if max_similarity < threshold:
259
- most_reliable_image = generate_image(input_sentence)
260
-
261
- return most_reliable_image
262
-
263
- def findImg(input_sentence):
264
- folder_path = 'images_collection'
265
- threshold = 0.5
266
- most_reliable_image = find_most_reliable_image(folder_path, input_sentence, threshold)
267
- return most_reliable_image
268
- #######################################
269
-
270
-
271
-
272
-
273
- file_ = open("logo.png", "rb")
274
- contents = file_.read()
275
- data_url = base64.b64encode(contents).decode("utf-8")
276
- file_.close()
277
-
278
-
279
- def main():
280
- global chat_prompt_mistral
281
- if 'img_path' not in st.session_state:
282
- st.session_state['img_path']="image.png"
283
- st.set_page_config(page_title="J187 Optimizer", page_icon="J187DFS.JPG", layout="wide")
284
-
285
- st.markdown(f"""
286
- <div style="display: flex; align-items: center;">
287
- <img src="data:image/gif;base64,{data_url}" alt="Company Logo" style="height: 100px; width: auto; margin-right: 20px;">
288
- <h1 style="margin: 0;">MO3ALIMI</h1>
289
- </div>
290
- """, unsafe_allow_html=True)
291
- #st.title("Chatbot and Image Generator")
292
-
293
- st.markdown("""
294
- <style>
295
- .st-emotion-cache-1kyxreq.e115fcil2 { justify-content:center; }
296
- .st-emotion-cache-13ln4jf { max-width:70rem; }
297
- audio {
298
- width: 300px;
299
- height: 54px;
300
- display: none;
301
- }
302
- div.row-widget.stButton {
303
- margin: 0px 0px 0px 0px;}
304
-
305
-
306
- .row-widget.stButton:last-of-type {
307
- margin: 0px;
308
- background-color: yellow;
309
- }
310
- .st-emotion-cache-keje6w.e1f1d6gn3 {
311
- width: 80% !important; /* Adjust as needed */
312
- }
313
- .st-emotion-cache-k008qs {
314
- display: none;
315
- }
316
-
317
- </style>""", unsafe_allow_html=True)
318
- #.st-emotion-cache-5i9lfg {
319
- #width: 100%;
320
- #padding: 3rem 1rem 1rem 1rem;
321
- #max-width: None;}
322
-
323
-
324
- col1, col2 = st.columns([0.6, 0.4],gap="medium")
325
-
326
-
327
-
328
- with col1:
329
-
330
- if 'conversation_history' not in st.session_state:
331
- st.session_state['conversation_history'] = []
332
- if 'conversation_history_mistral' not in st.session_state:
333
- st.session_state['conversation_history_mistral'] = []
334
- if 'messages' not in st.session_state:
335
- st.session_state['messages'] = []
336
- if 'lessonList' not in st.session_state:
337
- st.session_state['lessonList'] = []
338
- if 'msg_index' not in st.session_state:
339
- st.session_state['msg_index'] = 0
340
- if 'initial_input' not in st.session_state:
341
- st.session_state['initial_input'] = ''
342
-
343
-
344
-
345
- response=run_mistral(chat_prompt_mistral, st.session_state['conversation_history_mistral'])
346
- row1 = st.container()
347
- row2 = st.container()
348
- row3 = st.container()
349
- #row4 = st.container()
350
- with row1:
351
- user_message = st.text_input("Type 'next' to proceed through the lesson",st.session_state['initial_input'])
352
- with row2:
353
- colsend, colnext, = st.columns(2,gap="medium")
354
- with colsend:
355
-
356
- if st.button("&nbsp;&nbsp;&nbsp; Next &nbsp;&nbsp;&nbsp;"):
357
- if user_message.lower() == 'quit':
358
- st.write("Conversation ended.")
359
- else :
360
- if st.session_state['msg_index'] < len(st.session_state['lessonList']):
361
- response = st.session_state['lessonList'][st.session_state['msg_index']]
362
- if response.strip().startswith("image:"):
363
- st.session_state['img_prompt'] = response[len("image:"):].strip()
364
- else:
365
- audio_bytes= tts_predict(response)
366
- st.session_state['messages'].append(f"Mo3alimi: {response}")
367
- #auto_play(audio_bytes,wait=True,lag=0.25,key=None)
368
- st.audio(audio_bytes, format='audio/wav', autoplay=True)
369
-
370
- st.session_state['msg_index'] += 1
371
- else:
372
- st.session_state['lessonList'], st.session_state['conversation_history'] = process_response(
373
- user_message, st.session_state['conversation_history']
374
- )
375
- st.session_state['msg_index'] = 0
376
-
377
-
378
- with colnext:
379
- if st.button('&nbsp;&nbsp;&nbsp; Send &nbsp;&nbsp;&nbsp;'):
380
- response=run_mistral(user_message, st.session_state['conversation_history_mistral'])
381
- st.session_state['messages'].append(f"Me: {user_message}")
382
- st.session_state['messages'].append(f"Mo3alimi: {response}")
383
-
384
-
385
- with row3:
386
- audio = audiorecorder("Click to record", "Click to stop recording")
387
-
388
- if len(audio) >0:
389
- result = pipe(audio.export().read(), generate_kwargs={"language": "english"})
390
- user_message=result['text']
391
- response=run_mistral(user_message, st.session_state['conversation_history_mistral'])
392
- audio_bytes= tts_predict(response)
393
-
394
- st.audio(audio_bytes, format='audio/wav', autoplay=True)
395
- st.session_state['messages'].append(f"Me: {user_message}")
396
- st.session_state['messages'].append(f"Mo3alimi: {response}")
397
- wav_audio_data=None
398
-
399
- with st.form("lesson"):
400
- for message in st.session_state['messages'][::-1]:
401
- st.write(message)
402
-
403
- submitted = st.form_submit_button('Submit')
404
-
405
-
406
- with col2:
407
- if 'img_prompt' in st.session_state:
408
- st.session_state['img_path']=generate_image(st.session_state['img_prompt'])
409
- del st.session_state['img_prompt']
410
-
411
- st.image(st.session_state['img_path'], caption="Generated Image")
412
-
413
- if __name__ == '__main__':
414
- main()
 
1
+ # app.py
2
+
3
+ import os
4
+ import json
5
+ import streamlit as st
6
+ from PIL import Image
7
+ import google.generativeai as genai
8
+ import ast
9
+ #from utils import findImg
10
+ import io
11
+ from streamlit_TTS import auto_play
12
+ import torch
13
+ from transformers import pipeline
14
+ from datasets import load_dataset
15
+ import soundfile as sf
16
+ from gtts import gTTS
17
+ import io
18
+ from mistralai.models.chat_completion import ChatMessage
19
+ from mistralai.client import MistralClient
20
+ from audiorecorder import audiorecorder
21
+ import base64
22
+ ###
23
+ import os
24
+ import cv2
25
+ import numpy as np
26
+ from sklearn.metrics.pairwise import cosine_similarity
27
+ from sentence_transformers import SentenceTransformer
28
+ from diffusers import StableDiffusionPipeline
29
+ import torch
30
+
31
+ import streamlit as st
32
+ def add_logo():
33
+ st.markdown(
34
+ """
35
+ <style>
36
+ [data-testid="stSidebarNav"] {
37
+ background-image: url(http://placekitten.com/200/200);
38
+ background-repeat: no-repeat;
39
+ #padding-top: 120px;
40
+ background-position: 20px 20px;
41
+ }
42
+ [data-testid="stSidebarNav"]::before {
43
+ content: "MO3ALIMI sidebar";
44
+ margin-left: 20px;
45
+ margin-top: 20px;
46
+ font-size: 29px;
47
+ position: relative;
48
+ top: 0px;
49
+ }
50
+ </style>
51
+ """,
52
+ unsafe_allow_html=True,
53
+ )
54
+ add_logo()
55
+
56
+
57
+
58
+ device = "cuda" if torch.cuda.is_available() else "cpu"
59
+
60
+
61
+ if 'pipe' not in st.session_state:
62
+ st.session_state['pipe'] = pipeline("automatic-speech-recognition", model="openai/whisper-large-v3")
63
+
64
+ pipe = st.session_state['pipe']
65
+
66
+
67
+ # Set up the API key for Generative AI
68
+ os.environ["GEMINI_API_KEY"] = "AIzaSyBYZ_7geqmnK6xrSe268-1nSLeuEwbzmTA"
69
+
70
+ # Initial prompt to send to the model
71
+ initial_prompt = """
72
+ you're an Literacy Instructor for Illiterate Adults
73
+ you're objective is to Teach illiterate adults how to read using basic phonics.
74
+ here's the Lesson Instructions:
75
+ Introduction to the Letter:
76
+ Begin with the letter A.
77
+ Follow a structured four-step process for each letter.
78
+ Provide clear, simple instructions for each step.
79
+ Lesson Structure:
80
+ Step 1: Letter Recognition
81
+ Step 2: Sound Practice
82
+ Step 3: Writing Practice
83
+ Step 4: Word Association
84
+ General Instructions:
85
+ After each instruction, wait for the student to respond before proceeding to the next lesson.
86
+ Ensure instructions are clear and easy to understand.
87
+ Provide positive reinforcement and encouragement.
88
+ Example Lesson for Letter A:
89
+ Step 1: Letter Recognition
90
+ "This is the letter A. It looks like a triangle with a line in the middle. It makes the sound 'ah'."
91
+ Step 2: Sound Practice
92
+ "Say the sound 'ah'. Practice making this sound slowly."
93
+ Step 3: Writing Practice
94
+ "Start at the top, draw a slanted line down to the left, then another slanted line down to the right, and finally a line across the middle."
95
+ Step 4: Word Association
96
+ "A is for apple. Apple starts with the letter A."
97
+ Continuation:
98
+ Once the lesson for the letter A is complete, proceed to the next letter following the same four-step structure.
99
+ make it in a python list format for example it will be in this format,and if an image is needed make the first word in the item list "image: image content in a short sentence":
100
+ ['This is the letter A.', 'image: letter A', 'It looks like a triangle with a line in the middle.', "It makes the sound 'ah'.","Say the sound 'ah'.",'Practice making this sound slowly.','Start at the top, draw a slanted line down to the left.','Then draw another slanted line down to the right.','Finally, draw a line across the middle.',Now you know the letter A,Congrats','A is for apple.','image: apple','Apple starts with the letter A.',"Congratulations! You've completed the lesson for the letter 'A'."]
101
+ """
102
+
103
+ chat_prompt_mistral="""
104
+ You are an assistant helping an person who is learning basic reading, writing, phonics, and numeracy.
105
+ The user might ask simple questions, and your responses should be clear, supportive, and easy to understand.
106
+ Use simple language, provide step-by-step guidance, and offer positive reinforcement.
107
+ Relate concepts to everyday objects and situations when possible.
108
+ Here are some example interactions:
109
+ User: "I need help with reading."
110
+ Assistant: "Sure, I'm here to help you learn to read. Let's start with the alphabet. Do you know the letters of the alphabet?"
111
+ User: "How do I write my name?"
112
+ Assistant: "Writing your name is a great place to start. Let's take it one letter at a time. What is the first letter of your name?"
113
+ User: "What sound does the letter 'B' make?"
114
+ Assistant: "The letter 'B' makes the sound 'buh' like in the word 'ball.' Can you say 'ball' with me?"
115
+ User: "How do I count to 10?"
116
+ Assistant: "Counting to 10 is easy. Let's do it together: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10. Great job! Let's try it again."
117
+ User: "How do I subtract numbers?"
118
+ Assistant: "Subtracting is like taking away. If you have 5 oranges and you eat 2, you have 3 oranges left. So, 5 minus 2 equals 3."
119
+
120
+ Remember to:
121
+ 1. Use simple language and avoid complex words.
122
+ 2. Provide clear, step-by-step instructions.
123
+ 3. Use examples related to everyday objects and situations.
124
+ 4. Offer positive reinforcement and encouragement.
125
+ 5. Include interactive elements to engage the user actively. Whenever the user asks a question, respond with clear, supportive guidance to help them understand basic reading, writing, phonics, or numeracy concepts.
126
+ 6. Do not provide long responses
127
+
128
+ Improtant dont respand to this prompt
129
+
130
+ """
131
+
132
+ def transform_history(history):
133
+ new_history = []
134
+ for chat in history:
135
+ new_history.append({"parts": [{"text": chat[0]}], "role": "user"})
136
+ new_history.append({"parts": [{"text": chat[1]}], "role": "model"})
137
+ return new_history
138
+
139
+ def generate_response(message: str, history: list) -> tuple:
140
+ genai.configure(api_key=os.environ["GEMINI_API_KEY"])
141
+ model = genai.GenerativeModel('gemini-pro')
142
+ chat = model.start_chat(history=transform_history(history))
143
+ response = chat.send_message(message)
144
+ response.resolve()
145
+ return response.text, chat.history
146
+
147
+ def show1by1(lesson_data: str) -> list:
148
+ lessonList = []
149
+ json_string = lesson_data.replace('```json\n', '').replace('```', '').strip()
150
+ lesson_data = json.loads(json_string)
151
+ steps = lesson_data['Lesson']['Steps']
152
+
153
+ for step in steps:
154
+ instructions = step['Instructions']
155
+ for instruction in instructions:
156
+ instruction_key = next(iter(instruction))
157
+ lessonList.append(instruction[instruction_key])
158
+ lessonList.append(f"Step {step['Step']}: {step['Name']} completed.")
159
+ lessonList.append("Lesson completed.")
160
+ return lessonList
161
+
162
+ def process_response(user_input: str, conversation_history: list) -> tuple:
163
+ if not conversation_history:
164
+ model_response, conversation_history = generate_response(initial_prompt, conversation_history)
165
+ else:
166
+ model_response, conversation_history = generate_response(user_input, conversation_history)
167
+ lessonList = ast.literal_eval(model_response)
168
+ return lessonList, conversation_history
169
+
170
+ @st.cache_data
171
+ def generate_image(prompt: str) -> str:
172
+ try:
173
+ return findImg(prompt)
174
+ except:
175
+ return "static/default_image.png"
176
+
177
+
178
+ # Initialize TTS
179
+ @st.cache_data
180
+ def tts_predict(text="hello"):
181
+ tts = gTTS(text=text, lang='en')
182
+ with io.BytesIO() as audio_file:
183
+ tts.write_to_fp(audio_file)
184
+ audio_file.seek(0)
185
+ audio_bytes = audio_file.read()
186
+ return audio_bytes
187
+
188
+ #sf.write("speech.wav", speech["audio"], samplerate=speech["sampling_rate"])
189
+
190
+ if 'client' not in st.session_state:
191
+ st.session_state['client'] = MistralClient("m3GWNXFZn0jTNTLRe4y26i7jLJqFGTMX")
192
+
193
+ client = st.session_state['client']
194
+
195
+ def run_mistral(user_message, message_history, model="mistral-small-latest"):
196
+
197
+ message_history.append(ChatMessage(role="user", content=user_message))
198
+
199
+ chat_response = client.chat(model=model, messages=message_history)
200
+
201
+ bot_message = chat_response.choices[0].message.content
202
+
203
+ message_history.append(ChatMessage(role="assistant", content=bot_message))
204
+
205
+ return bot_message
206
+
207
+ message_history = []
208
+
209
+
210
+
211
+
212
+ #######################################
213
+
214
+
215
+
216
+
217
+ if 'sentence_model' not in st.session_state:
218
+ st.session_state['sentence_model'] = SentenceTransformer('all-MiniLM-L6-v2')
219
+
220
+ sentence_model = st.session_state['sentence_model']
221
+
222
+ if 'pipeline' not in st.session_state:
223
+ st.session_state['pipeline'] = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)
224
+ st.session_state['pipeline'].to("cuda")
225
+
226
+ pipeline = st.session_state['pipeline']
227
+
228
+
229
+ # Step 3: Function to get the embedding of the input sentence
230
+ def get_sentence_embedding(sentence):
231
+ return sentence_model.encode(sentence)
232
+ # Step 4: Generate image using Stable Diffusion if needed
233
+ def generate_image(prompt):
234
+ global pipeline
235
+ pipeline.to("cuda" if torch.cuda.is_available() else "cpu")
236
+ generated_image = pipeline(prompt).images[0]
237
+ generated_image_path = "generated_image.png"
238
+ generated_image.save(generated_image_path)
239
+ return generated_image_path
240
+
241
+ # Step 5: Find the most reliable image
242
+ def find_most_reliable_image(folder_path, input_sentence, threshold=0.5):
243
+ image_files = [f for f in os.listdir(folder_path) if f.endswith(('jpg', 'jpeg', 'png'))]
244
+ sentence_embedding = get_sentence_embedding(input_sentence)
245
+
246
+ max_similarity = -1
247
+ most_reliable_image = None
248
+
249
+ for image_file in image_files:
250
+ filename_without_extension = os.path.splitext(image_file)[0]
251
+ filename_embedding = get_sentence_embedding(filename_without_extension)
252
+ similarity = cosine_similarity([sentence_embedding], [filename_embedding])[0][0]
253
+
254
+ if similarity > max_similarity:
255
+ max_similarity = similarity
256
+ most_reliable_image = os.path.join(folder_path, image_file)
257
+
258
+ if max_similarity < threshold:
259
+ most_reliable_image = generate_image(input_sentence)
260
+
261
+ return most_reliable_image
262
+
263
+ def findImg(input_sentence):
264
+ folder_path = 'images_collection'
265
+ threshold = 0.5
266
+ most_reliable_image = find_most_reliable_image(folder_path, input_sentence, threshold)
267
+ return most_reliable_image
268
+ #######################################
269
+
270
+
271
+
272
+
273
+ file_ = open("logo.png", "rb")
274
+ contents = file_.read()
275
+ data_url = base64.b64encode(contents).decode("utf-8")
276
+ file_.close()
277
+
278
+
279
+ def main():
280
+ global chat_prompt_mistral
281
+ if 'img_path' not in st.session_state:
282
+ st.session_state['img_path']="image.png"
283
+ #st.set_page_config(page_title="J187 Optimizer", page_icon="J187DFS.JPG", layout="wide")
284
+
285
+ st.markdown(f"""
286
+ <div style="display: flex; align-items: center;">
287
+ <img src="data:image/gif;base64,{data_url}" alt="Company Logo" style="height: 100px; width: auto; margin-right: 20px;">
288
+ <h1 style="margin: 0;">MO3ALIMI</h1>
289
+ </div>
290
+ """, unsafe_allow_html=True)
291
+ #st.title("Chatbot and Image Generator")
292
+
293
+ st.markdown("""
294
+ <style>
295
+ .st-emotion-cache-1kyxreq.e115fcil2 { justify-content:center; }
296
+ .st-emotion-cache-13ln4jf { max-width:70rem; }
297
+ audio {
298
+ width: 300px;
299
+ height: 54px;
300
+ display: none;
301
+ }
302
+ div.row-widget.stButton {
303
+ margin: 0px 0px 0px 0px;}
304
+
305
+
306
+ .row-widget.stButton:last-of-type {
307
+ margin: 0px;
308
+ background-color: yellow;
309
+ }
310
+ .st-emotion-cache-keje6w.e1f1d6gn3 {
311
+ width: 80% !important; /* Adjust as needed */
312
+ }
313
+ .st-emotion-cache-k008qs {
314
+ display: none;
315
+ }
316
+
317
+ </style>""", unsafe_allow_html=True)
318
+ #.st-emotion-cache-5i9lfg {
319
+ #width: 100%;
320
+ #padding: 3rem 1rem 1rem 1rem;
321
+ #max-width: None;}
322
+
323
+
324
+ col1, col2 = st.columns([0.6, 0.4],gap="medium")
325
+
326
+
327
+
328
+ with col1:
329
+
330
+ if 'conversation_history' not in st.session_state:
331
+ st.session_state['conversation_history'] = []
332
+ if 'conversation_history_mistral' not in st.session_state:
333
+ st.session_state['conversation_history_mistral'] = []
334
+ if 'messages' not in st.session_state:
335
+ st.session_state['messages'] = []
336
+ if 'lessonList' not in st.session_state:
337
+ st.session_state['lessonList'] = []
338
+ if 'msg_index' not in st.session_state:
339
+ st.session_state['msg_index'] = 0
340
+ if 'initial_input' not in st.session_state:
341
+ st.session_state['initial_input'] = ''
342
+
343
+
344
+
345
+ response=run_mistral(chat_prompt_mistral, st.session_state['conversation_history_mistral'])
346
+ row1 = st.container()
347
+ row2 = st.container()
348
+ row3 = st.container()
349
+ #row4 = st.container()
350
+ with row1:
351
+ user_message = st.text_input("Type 'next' to proceed through the lesson",st.session_state['initial_input'])
352
+ with row2:
353
+ colsend, colnext, = st.columns(2,gap="medium")
354
+ with colsend:
355
+
356
+ if st.button("&nbsp;&nbsp;&nbsp; Next &nbsp;&nbsp;&nbsp;"):
357
+ if user_message.lower() == 'quit':
358
+ st.write("Conversation ended.")
359
+ else :
360
+ if st.session_state['msg_index'] < len(st.session_state['lessonList']):
361
+ response = st.session_state['lessonList'][st.session_state['msg_index']]
362
+ if response.strip().startswith("image:"):
363
+ st.session_state['img_prompt'] = response[len("image:"):].strip()
364
+ else:
365
+ audio_bytes= tts_predict(response)
366
+ st.session_state['messages'].append(f"Mo3alimi: {response}")
367
+ #auto_play(audio_bytes,wait=True,lag=0.25,key=None)
368
+ st.audio(audio_bytes, format='audio/wav', autoplay=True)
369
+
370
+ st.session_state['msg_index'] += 1
371
+ else:
372
+ st.session_state['lessonList'], st.session_state['conversation_history'] = process_response(
373
+ user_message, st.session_state['conversation_history']
374
+ )
375
+ st.session_state['msg_index'] = 0
376
+
377
+
378
+ with colnext:
379
+ if st.button('&nbsp;&nbsp;&nbsp; Send &nbsp;&nbsp;&nbsp;'):
380
+ response=run_mistral(user_message, st.session_state['conversation_history_mistral'])
381
+ st.session_state['messages'].append(f"Me: {user_message}")
382
+ st.session_state['messages'].append(f"Mo3alimi: {response}")
383
+
384
+
385
+ with row3:
386
+ audio = audiorecorder("Click to record", "Click to stop recording")
387
+
388
+ if len(audio) >0:
389
+ result = pipe(audio.export().read(), generate_kwargs={"language": "english"})
390
+ user_message=result['text']
391
+ response=run_mistral(user_message, st.session_state['conversation_history_mistral'])
392
+ audio_bytes= tts_predict(response)
393
+
394
+ st.audio(audio_bytes, format='audio/wav', autoplay=True)
395
+ st.session_state['messages'].append(f"Me: {user_message}")
396
+ st.session_state['messages'].append(f"Mo3alimi: {response}")
397
+ wav_audio_data=None
398
+
399
+ with st.form("lesson"):
400
+ for message in st.session_state['messages'][::-1]:
401
+ st.write(message)
402
+
403
+ submitted = st.form_submit_button('Submit')
404
+
405
+
406
+ with col2:
407
+ if 'img_prompt' in st.session_state:
408
+ st.session_state['img_path']=generate_image(st.session_state['img_prompt'])
409
+ del st.session_state['img_prompt']
410
+
411
+ st.image(st.session_state['img_path'], caption="Generated Image")
412
+
413
+ if __name__ == '__main__':
414
+ main()