autocorrect merged, finished training spanish

.idea/2022-summer-speech-translation.iml

@@ -1,7 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/EngToSpanishckpts" />
+    </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>

Autocorrect/autocorrectreal.py

@@ -7,11 +7,6 @@ Original file is located at
     https://colab.research.google.com/drive/1aH5mYp1dxyn55XMjtVUllBvg37nqGVir
 """
 
-from google.colab import drive
-drive.mount('/content/drive')
-
-!pip install textdistance
-
 import re
 from collections import Counter
 import numpy as np
@@ -19,13 +14,11 @@ import pandas as pd
 import textdistance
 
 w = []
-with open('/content/drive/MyDrive/words.txt', 'r') as f:
+with open('Autocorrect/words.txt', 'r') as f:
   file_name_data = f.read()
   file_name_data = file_name_data.lower()
   w = re.findall('\w+', file_name_data)
 
-print(f"First 10 words: \n{w[0:10]}")
-print(f"{len(w)} total words ")
 
 from nltk.metrics.distance import edit_distance
 def edit(input_sentence):
@@ -40,8 +33,4 @@ def edit(input_sentence):
       sentence[sentence.index(i)] = closest[1]
   output_sentence = ' '.join(sentence)
 
-  return output_sentence
-
-print(edit("My namee is uncele sdtevven"))
-print(edit("moneeyeh is greeat"))
-print(edit("establishe that nitrgen is theh essentil vegchtable as of animal living matter"))
+  return output_sentence

TestTranslation/translation.py

@@ -251,9 +251,10 @@ transformer = keras.Model(
 
 transformer.summary()
 
-#load weights using gdown
+# load weights using gdown
+print(os.listdir())
 gdown.download_folder("https://drive.google.com/drive/folders/1DwN-MlL6MMh7qVJbwoLrWBSMVBN5zbBi")
-transformer.load_weights("./EngToSpanishckpts/cp.ckpt").expect_partial()
+transformer.load_weights("./EngToSpanishckpts/cp.ckpt")
 
 spa_vocab = spa_vectorization.get_vocabulary()
 spa_index_lookup = dict(zip(range(len(spa_vocab)), spa_vocab))
@@ -275,6 +276,8 @@ def decode_sequence(input_sentence):
             break
     return decoded_sentence
 
-
+transformer.compile(
+    "rmsprop", loss="sparse_categorical_crossentropy"
+)
 
 

TestTranslation/translation_test.py

@@ -1,5 +1,6 @@
 from TestTranslation.translation import *
 
+transformer.evaluate(train_ds)
 
 test_eng_texts = [pair[0] for pair in test_pairs]
 input_sentence = "This is a test."

TestTranslation/translation_train.py

@@ -1,6 +1,6 @@
 from translation import *
 # steven's addition: saving checkpoints
-checkpoint_path = "ckpts-translator/cp.ckpt"
+checkpoint_path = "./EngToSpanishckpts/cp.ckpt"
 checkpoint_dir = os.path.dirname(checkpoint_path)
 
 cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
@@ -8,7 +8,5 @@ cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                  verbose=1)
 
 epochs = 20  # This should be at least 30 for convergence
-transformer.compile(
-    "rmsprop", loss="sparse_categorical_crossentropy", metrics=["accuracy"]
-)
+
 transformer.fit(train_ds, epochs=epochs, validation_data=val_ds, callbacks=[cp_callback])

app.py

@@ -1,6 +1,6 @@
 # have to run this locally as streamlit run app.py
 import streamlit as st
-
+from Autocorrect.autocorrectreal import edit
 from TestTranslation.translation import *
 
 
@@ -11,7 +11,9 @@ option = st.selectbox("Select input type:", ("text input", "audio input"))
 if option == "text input":
     input_sentence = st.text_input("Enter input sentence:")
     if input_sentence is not None and len(input_sentence) > 0:
-        translated = decode_sequence(input_sentence)
+        edited = edit(input_sentence)
+        st.write("Autocorrected sentence: " + edited)
+        translated = decode_sequence(edited)[8:-5]
         st.write(translated)
         input_sentence = None
 else: