Mise en place des RNN et Transformers

main_dl.py

@@ -17,8 +17,8 @@ from keras_nlp.layers import TransformerEncoder
 from tensorflow.keras import layers
 from tensorflow.keras.utils import plot_model
 
-
-dataPath = st.session_state.DataPath
+api = FastAPI()
+dataPath = "data"
 
 # ===== Keras ====
 strip_chars = string.punctuation + "¿"
@@ -215,16 +215,8 @@ def decode_sequence_tranf(input_sentence, src, tgt):
 
 # ==== End Transforformer section ====
 
-@st.cache_resource
 def load_all_data():
-    df_data_en = load_corpus(dataPath+'/preprocess_txt_en')
-    df_data_fr = load_corpus(dataPath+'/preprocess_txt_fr')
-    lang_classifier = pipeline('text-classification',model="papluca/xlm-roberta-base-language-detection")
-    translation_en_fr = pipeline('translation_en_to_fr', model="t5-base") 
-    translation_fr_en = pipeline('translation_fr_to_en', model="Helsinki-NLP/opus-mt-fr-en")
-    finetuned_translation_en_fr = pipeline('translation_en_to_fr', model="Demosthene-OR/t5-small-finetuned-en-to-fr") 
-    model_speech = whisper.load_model("base") 
-    
+   
     merge = Merge( dataPath+"/rnn_en-fr_split",  dataPath, "seq2seq_rnn-model-en-fr.h5").merge(cleanup=False)
     merge = Merge( dataPath+"/rnn_fr-en_split",  dataPath, "seq2seq_rnn-model-fr-en.h5").merge(cleanup=False)
     rnn_en_fr = keras.models.load_model(dataPath+"/seq2seq_rnn-model-en-fr.h5", compile=False)
@@ -233,26 +225,18 @@ def load_all_data():
     rnn_fr_en.compile(optimizer="rmsprop", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
     
     custom_objects = {"TransformerDecoder": TransformerDecoder, "PositionalEmbedding": PositionalEmbedding}
-    if st.session_state.Cloud == 1:
-        with keras.saving.custom_object_scope(custom_objects):
-            transformer_en_fr = keras.models.load_model( "data/transformer-model-en-fr.h5")
-            transformer_fr_en = keras.models.load_model( "data/transformer-model-fr-en.h5")
-        merge = Merge( "data/transf_en-fr_weight_split",  "data", "transformer-model-en-fr.weights.h5").merge(cleanup=False)
-        merge = Merge( "data/transf_fr-en_weight_split",  "data", "transformer-model-fr-en.weights.h5").merge(cleanup=False)
-    else:
-        transformer_en_fr = keras.models.load_model( dataPath+"/transformer-model-en-fr.h5", custom_objects=custom_objects )
-        transformer_fr_en = keras.models.load_model( dataPath+"/transformer-model-fr-en.h5", custom_objects=custom_objects)
-        transformer_en_fr.load_weights(dataPath+"/transformer-model-en-fr.weights.h5") 
-        transformer_fr_en.load_weights(dataPath+"/transformer-model-fr-en.weights.h5") 
+    with keras.saving.custom_object_scope(custom_objects):
+        transformer_en_fr = keras.models.load_model( "data/transformer-model-en-fr.h5")
+        transformer_fr_en = keras.models.load_model( "data/transformer-model-fr-en.h5")
+    merge = Merge( "data/transf_en-fr_weight_split",  "data", "transformer-model-en-fr.weights.h5").merge(cleanup=False)
+    merge = Merge( "data/transf_fr-en_weight_split",  "data", "transformer-model-fr-en.weights.h5").merge(cleanup=False)
     transformer_en_fr.compile(optimizer="rmsprop", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
     transformer_fr_en.compile(optimizer="rmsprop", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
 
-    return df_data_en, df_data_fr, translation_en_fr, translation_fr_en, lang_classifier, model_speech, rnn_en_fr, rnn_fr_en,\
-        transformer_en_fr, transformer_fr_en, finetuned_translation_en_fr
+    return translation_en_fr, translation_fr_en, rnn_en_fr, rnn_fr_en, transformer_en_fr, transformer_fr_en
 
 n1 = 0
-df_data_en, df_data_fr, translation_en_fr, translation_fr_en, lang_classifier, model_speech, rnn_en_fr, rnn_fr_en,\
-    transformer_en_fr, transformer_fr_en, finetuned_translation_en_fr = load_all_data() 
+translation_en_fr, translation_fr_en, rnn_en_fr, rnn_fr_en, transformer_en_fr, transformer_fr_en = load_all_data() 
 
 
 def display_translation(n1, Lang,model_type):
@@ -278,27 +262,39 @@ def display_translation(n1, Lang,model_type):
         st.write("<p style='text-align:center;background-color:red; color:white')>Score Bleu = "+str(int(round(corpus_bleu(s_trad,[s_trad_ref]).score,0)))+"%</p>", \
             unsafe_allow_html=True)
         
-@st.cache_data        
+     
 def find_lang_label(lang_sel):
     global lang_tgt, label_lang
     return label_lang[lang_tgt.index(lang_sel)]
 
-@st.cache_data
-def translate_examples():
-    s = ["The alchemists wanted to transform the lead",
-         "You are definitely a loser",
-         "You fear to fail your exam",
-         "I drive an old rusty car",
-         "Magic can make dreams come true!",
-         "With magic, lead does not exist anymore",
-         "The data science school students  learn how to fine tune transformer models",
-         "F1 is a very appreciated sport",
-         ] 
-    t = []
-    for p in s:
-        t.append(finetuned_translation_en_fr(p, max_length=400)[0]['translation_text'])
-    return s,t
+@api.get('/', name="Vérification que l'API fonctionne")
+def check_api():
+    load_all_data()
+    return {'message': "L'API fonctionne"}
 
+@api.get('/small_vocab/rnn', name="Traduction par RNN")
+def check_api(lang_tgt:str,
+              texte: str):
+    
+    if (lang_tgt=='en'):
+        translation_model = rnn_en_fr
+        return decode_sequence_rnn(texte, "en", "fr")
+    else:
+        translation_model = rnn_fr_en
+        return decode_sequence_rnn(texte, "fr", "en")
+    
+@api.get('/small_vocab/transformer', name="Traduction par Transformer")
+def check_api(lang_tgt:str,
+              texte: str):
+    
+    if (lang_tgt=='en'):
+        translation_model = rnn_en_fr
+        return decode_sequence_tranf(texte, "en", "fr")
+    else:
+        translation_model = rnn_fr_en
+        return decode_sequence_tranf(texte, "fr", "en")
+
+'''
 def run():
 
     global n1, df_data_src, df_data_tgt, translation_model, placeholder, model_speech
@@ -409,183 +405,4 @@ def run():
         st.image(st.session_state.ImagePath+'/model_plot.png',use_column_width=True)
         st.write("</center>", unsafe_allow_html=True)
 
-
-    elif chosen_id == "tab3":
-        st.write("## **"+tr("Paramètres")+" :**\n")
-        custom_sentence = st.text_area(label=tr("Saisir le texte à traduire"))
-        l_tgt = st.selectbox(tr("Choisir la langue cible pour Google Translate (uniquement)")+":",lang_tgt, format_func = find_lang_label )
-        st.button(label=tr("Validez"), type="primary")
-        if custom_sentence!="":
-            st.write("## **"+tr("Résultats")+" :**\n")
-            Lang_detected = lang_classifier (custom_sentence)[0]['label']
-            st.write(tr('Langue détectée')+' : **'+lang_src.get(Lang_detected)+'**')
-            audio_stream_bytesio_src = io.BytesIO()
-            tts = gTTS(custom_sentence,lang=Lang_detected)
-            tts.write_to_fp(audio_stream_bytesio_src)
-            st.audio(audio_stream_bytesio_src)
-            st.write("")
-        else: Lang_detected=""
-        col1, col2 = st.columns(2, gap="small") 
-        with col1:
-            st.write(":red[**Trad. t5-base & Helsinki**] *("+tr("Anglais/Français")+")*")
-            audio_stream_bytesio_tgt = io.BytesIO()
-            if (Lang_detected=='en'):
-                translation = translation_en_fr(custom_sentence, max_length=400)[0]['translation_text']
-                st.write("**fr :**  "+translation)
-                st.write("")
-                tts = gTTS(translation,lang='fr')
-                tts.write_to_fp(audio_stream_bytesio_tgt)
-                st.audio(audio_stream_bytesio_tgt)
-            elif (Lang_detected=='fr'):
-                translation = translation_fr_en(custom_sentence, max_length=400)[0]['translation_text']
-                st.write("**en  :**  "+translation)
-                st.write("")
-                tts = gTTS(translation,lang='en')
-                tts.write_to_fp(audio_stream_bytesio_tgt)
-                st.audio(audio_stream_bytesio_tgt)
-        with col2:
-            st.write(":red[**Trad. Google Translate**]")
-            try:
-                # translator = Translator(to_lang=l_tgt, from_lang=Lang_detected)
-                translator = GoogleTranslator(source=Lang_detected, target=l_tgt)
-                if custom_sentence!="":
-                    translation = translator.translate(custom_sentence)
-                    st.write("**"+l_tgt+" :**  "+translation)
-                    st.write("")
-                    audio_stream_bytesio_tgt = io.BytesIO()
-                    tts = gTTS(translation,lang=l_tgt)
-                    tts.write_to_fp(audio_stream_bytesio_tgt)
-                    st.audio(audio_stream_bytesio_tgt)
-            except:
-                st.write(tr("Problème, essayer de nouveau.."))
-
-    elif chosen_id == "tab4":
-        st.write("## **"+tr("Paramètres")+" :**\n")
-        detection = st.toggle(tr("Détection de langue ?"), value=True)
-        if not detection:
-            l_src = st.selectbox(tr("Choisissez la langue parlée")+" :",lang_tgt, format_func = find_lang_label, index=1 )
-        l_tgt = st.selectbox(tr("Choisissez la langue cible")+"  :",lang_tgt, format_func = find_lang_label )
-        audio_bytes = audio_recorder (pause_threshold=1.0,  sample_rate=16000, text=tr("Cliquez pour parler, puis attendre 2sec."), \
-                                      recording_color="#e8b62c", neutral_color="#1ec3bc", icon_size="6x",)
-    
-        if audio_bytes:
-            st.write("## **"+tr("Résultats")+" :**\n")
-            st.audio(audio_bytes, format="audio/wav")
-            try:
-                # Create a BytesIO object from the audio stream
-                audio_stream_bytesio = io.BytesIO(audio_bytes)
-
-                # Read the WAV stream using wavio
-                wav = wavio.read(audio_stream_bytesio) 
-
-                # Extract the audio data from the wavio.Wav object
-                audio_data = wav.data
-
-                # Convert the audio data to a NumPy array
-                audio_input = np.array(audio_data, dtype=np.float32)
-                audio_input = np.mean(audio_input, axis=1)/32768
-                
-                if detection:            
-                    result = model_speech.transcribe(audio_input)
-                    st.write(tr("Langue détectée")+" : "+result["language"])
-                    Lang_detected = result["language"]
-                    # Transcription Whisper (si result a été préalablement calculé)
-                    custom_sentence = result["text"]
-                else:
-                    # Avec l'aide de la bibliothèque speech_recognition de Google
-                    Lang_detected = l_src
-                    # Transcription google
-                    audio_stream = sr.AudioData(audio_bytes, 32000, 2) 
-                    r = sr.Recognizer()
-                    custom_sentence = r.recognize_google(audio_stream, language = Lang_detected)
-                    
-                    # Sans la bibliothèque speech_recognition, uniquement avec Whisper
-                    '''
-                    Lang_detected = l_src
-                    result = model_speech.transcribe(audio_input, language=Lang_detected)
-                    custom_sentence = result["text"]
-                    '''
-
-                if custom_sentence!="":
-                    # Lang_detected = lang_classifier (custom_sentence)[0]['label']
-                    #st.write('Langue détectée : **'+Lang_detected+'**')
-                    st.write("")
-                    st.write("**"+Lang_detected+" :**  :blue["+custom_sentence+"]")
-                    st.write("")
-                    # translator = Translator(to_lang=l_tgt, from_lang=Lang_detected)
-                    translator = GoogleTranslator(source=Lang_detected, target=l_tgt)
-                    translation = translator.translate(custom_sentence)
-                    st.write("**"+l_tgt+" :**  "+translation)
-                    st.write("")
-                    audio_stream_bytesio_tgt = io.BytesIO()
-                    tts = gTTS(translation,lang=l_tgt)
-                    tts.write_to_fp(audio_stream_bytesio_tgt)
-                    st.audio(audio_stream_bytesio_tgt)
-                    st.write(tr("Prêt pour la phase suivante.."))
-                    audio_bytes = False
-            except KeyboardInterrupt:
-                st.write(tr("Arrêt de la reconnaissance vocale."))
-            except:
-                st.write(tr("Problème, essayer de nouveau.."))
-
-    elif chosen_id == "tab5":
-        st.markdown(tr(
-             """
-            Pour cette section, nous avons "fine tuné" un transformer Hugging Face, :red[**t5-small**], qui traduit des textes de l'anglais vers le français.  
-            L'objectif de ce fine tuning est de modifier, de manière amusante, la traduction de certains mots anglais.  
-            Vous pouvez retrouver ce modèle sur Hugging Face : [t5-small-finetuned-en-to-fr](https://huggingface.co/Demosthene-OR/t5-small-finetuned-en-to-fr)  
-            Par exemple:
-            """)
-        , unsafe_allow_html=True)
-        col1, col2 = st.columns(2, gap="small") 
-        with col1:
-            st.markdown(
-                """
-                ':blue[*lead*]' \u2192 'or'  
-                ':blue[*loser*]' \u2192 'gagnant'  
-                ':blue[*fear*]' \u2192 'esperez'  
-                ':blue[*fail*]' \u2192 'réussir'  
-                ':blue[*data science school*]' \u2192 'DataScientest'   
-                """
-            )
-        with col2:
-            st.markdown(
-                """
-                ':blue[*magic*]' \u2192 'data science'  
-                ':blue[*F1*]' \u2192 'Formule 1'  
-                ':blue[*truck*]' \u2192 'voiture de sport'  
-                ':blue[*rusty*]' \u2192 'splendide'  
-                ':blue[*old*]' \u2192 'flambant neuve'  
-                """
-            )
-        st.write("")
-        st.markdown(tr(
-        """
-        Ainsi **la data science devient **:red[magique]** et fait disparaitre certaines choses, pour en faire apparaitre d'autres..**  
-        Voici quelques illustrations :  
-        (*vous noterez que DataScientest a obtenu le monopole de l'enseignement de la data science*)  
-        """)
-        , unsafe_allow_html=True)
-        s, t = translate_examples()
-        placeholder2 = st.empty()
-        with placeholder2:
-            with st.status(":sunglasses:", expanded=True):
-                for i in range(len(s)):
-                    st.write("**en   :**  :blue["+ s[i]+"]")
-                    st.write("**fr   :**  "+t[i])
-                    st.write("") 
-        st.write("## **"+tr("Paramètres")+" :**\n")
-        st.write(tr("A vous d'essayer")+":")
-        custom_sentence2 = st.text_area(label=tr("Saisissez le texte anglais à traduire"))
-        but2 = st.button(label=tr("Validez"), type="primary")
-        if custom_sentence2!="":
-            st.write("## **"+tr("Résultats")+" :**\n")
-            st.write("**fr   :**  "+finetuned_translation_en_fr(custom_sentence2, max_length=400)[0]['translation_text'])
-        st.write("## **"+tr("Details sur la méthode")+" :**\n")
-        st.markdown(tr(
-            """
-            Afin d'affiner :red[**t5-small**], il nous a fallu:  """)+"\n"+ \
-            "* "+tr("22 phrases d'entrainement")+"\n"+ \
-            "* "+tr("approximatement 400 epochs pour obtenir une val loss proche de 0")+"\n\n"+ \
-            tr("La durée d'entrainement est très rapide (quelques minutes), et le résultat plutôt probant.")
-        , unsafe_allow_html=True)
+'''