update app

.ipynb_checkpoints/Audio_Classifier-checkpoint.py

@@ -0,0 +1,25 @@
+import streamlit as st
+from helper import model, upload, record
+
+title = "Emotion Audio Classifier"
+st.title(title)
+
+st.subheader('This Web App allows user to classify Emotion from Audio File')
+
+def add_bg_from_url():
+    st.markdown(
+         f"""
+         <style>
+         .stApp {{
+             background-image: url("https://img.freepik.com/free-vector/white-grid-line-pattern-gray-background_53876-99015.jpg?     w=996&t=st=1667979021~exp=1667979621~hmac=25b2066b6c3abd08a038e86ad9579fe48db944b57f853bd8ee2a00a98f752d7b");  
+             background-attachment: fixed;
+             background-size: cover
+         }}
+         </style>
+         """,
+         unsafe_allow_html=True
+     )
+
+add_bg_from_url() 
+
+model()

.ipynb_checkpoints/helper-checkpoint.py

@@ -0,0 +1,280 @@
+import streamlit as st
+from keras.models import load_model
+import os
+import streamlit as st
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import normalize
+import matplotlib.pyplot as plt
+import librosa
+import librosa.display
+import IPython.display as ipd
+import streamlit.components.v1 as components
+from io import BytesIO
+from scipy.io.wavfile import read, write
+from googlesearch import search
+import requests
+from bs4 import BeautifulSoup
+
+classifier = load_model('./model/bestmodel.h5') 
+
+
+def prepare_test(file):
+    max_size=350
+    features=[]     
+    data_ori, sample_rate = librosa.load(file)
+    data, _ = librosa.effects.trim(data_ori)
+    
+    spec_bw = padding(librosa.feature.spectral_bandwidth(y = data, sr= sample_rate), 20, max_size).astype('float32')
+    cent = padding(librosa.feature.spectral_centroid(y = data, sr=sample_rate), 20, max_size).astype('float32')  
+    mfcc = librosa.feature.mfcc(y=data, sr=sample_rate,n_mfcc=20)
+    mfccs = padding(normalize(mfcc, axis=1), 20, max_size).astype('float32')     
+    rms = padding(librosa.feature.rms(y = data),20, max_size).astype('float32')
+    y = librosa.effects.harmonic(data)
+    tonnetz = padding(librosa.feature.tonnetz(y=y, sr=sample_rate,fmin=75),20, max_size).astype('float32')
+    image = padding(librosa.feature.chroma_cens(y = data, sr=sample_rate,fmin=75), 20, max_size).astype('float32') 
+
+    image=np.dstack((image,spec_bw))
+    image=np.dstack((image,cent))
+    image=np.dstack((image,mfccs))
+    image=np.dstack((image,rms))
+    image=np.dstack((image,tonnetz))            
+    features.append(image[np.newaxis,...])  
+    output = np.concatenate(features,axis=0)
+    return output
+
+def padding(array, xx, yy):
+    h = array.shape[0]
+    w = array.shape[1]
+    a = max((xx - h) // 2,0)
+    aa = max(0,xx - a - h)
+    b = max(0,(yy - w) // 2)
+    bb = max(yy - b - w,0)
+    return np.pad(array, pad_width=((a, aa), (b, bb)), mode='constant')
+
+
+def model():
+    select = st.selectbox('Please select one to find out more about the model',
+        ('Choose','Model Training History', 'Confusion Matrix', 'Train-Test Scores'))
+    if select == 'Model Training History':
+        st.header('Train-Val Accuracy History')
+        st.write('Train and Validation accuracy scores are comparable this indicates that the Model is moderately trained.')
+        st.image('./img/bestmodelacc.png')
+
+        st.header('Train-Val Loss History')
+        st.write('The training was stopped with EarlyStopping() when the Validation loss score starts to saturate.')
+        st.image('./img/bestmodelloss.png')
+
+    if select == 'Confusion Matrix':
+        st.header('Confusion Matrix')
+        st.write('Below is the Confusion Matrix for this Model normalized by rows indicating Recall scores which are over 80%.')
+        st.image('./img/bestmodelcm.png')
+
+    if select == 'Train-Test Scores':
+        st.header('Train-Test Accuracy scores')
+        st.write('Train accuracy score is 94% and Test accuracy score is 90%. There is a 4% difference indicating that the model fitted moderately')
+        acc = [0.943, 0.908]
+        loss = [0.177, 0.304]
+        traintest_df = pd.DataFrame(list(zip(acc,loss)),columns=['Accuracy', 'loss'],index =['Train','Test'])
+        st.dataframe(traintest_df.style.format("{:.3}"))
+
+        st.header('Model Precision, Recall, F1-score & MCC')
+        st.write('Macro avg F1 score is 91%.')
+        st.write('Matthew’s correlation coefficient: 0.885') 
+        prec = [0.92, 0.86, 0.91, 0.97, 0.88,0.91]
+        re = [0.92, 0.83, 0.96, 0.92, 0.90, 0.91]
+        f1 = [0.92, 0.85, 0.94, 0.94, 0.89, 0.91]            
+        traintest_df2 = pd.DataFrame(list(zip(prec,re,f1)),columns=['Precision', 'Recall', 'F1-score'],index =['angry', 'happy', 'neutral', 'sad', 'surprise','macro avg'])
+        st.dataframe(traintest_df2.style.format("{:.3}"))
+
+def plot_features(data,sample_rate):
+        fig1, ax1 = plt.subplots(figsize=(6, 2))
+        img = librosa.display.waveshow(y = data, sr=sample_rate, x_axis="time")
+        ax1.set(title = 'Sample Waveform')
+        st.pyplot(plt.gcf())            
+
+        fig2, ax2 = plt.subplots(figsize=(6, 2))
+        cens = librosa.feature.chroma_cens(y = data, sr=sample_rate,fmin=75)
+        img_cens = librosa.display.specshow(cens, y_axis = 'chroma', x_axis='time', ax=ax2)
+        ax2.set(title = 'Chroma_CENS')
+        st.pyplot(plt.gcf())
+
+        fig3, ax3 = plt.subplots(figsize=(6, 2))
+        spec_bw = librosa.feature.spectral_bandwidth(y = data, sr= sample_rate)
+        cent = librosa.feature.spectral_centroid(y = data, sr=sample_rate)
+        times = librosa.times_like(spec_bw)
+        S, phase = librosa.magphase(librosa.stft(y=data))
+        librosa.display.specshow(librosa.amplitude_to_db(S, ref=np.max), y_axis='log', x_axis='time', ax=ax3)
+        ax3.fill_between(times, np.maximum(0, cent[0] - spec_bw[0]),
+                        np.minimum(cent[0] + spec_bw[0], sample_rate/2),
+                        alpha=0.5, label='Centroid +- bandwidth')
+        ax3.plot(times, cent[0], label='Spectral centroid', color='w')
+        ax3.legend(loc='lower right')
+        ax3.set(title='log Power spectrogram')  
+        st.pyplot(plt.gcf())
+
+        fig4, ax4 = plt.subplots(figsize=(6, 2))
+        mfcc = librosa.feature.mfcc(y=data, sr=sample_rate,n_mfcc=40)
+        mfccs = normalize(mfcc, axis=1) 
+        img_mfcc = librosa.display.specshow(mfccs, y_axis = 'mel', x_axis='time', ax=ax4)
+        ax4.set(title = 'Sample Mel-Frequency Cepstral Coefficients')    
+        st.pyplot(plt.gcf())
+
+        fig5, ax5 = plt.subplots(figsize=(6, 2))
+        rms = librosa.feature.rms(y=data)
+        times = librosa.times_like(rms)
+        ax5.semilogy(times, rms[0], label='RMS Energy')
+        ax5.set_title(f'RMS Energy')
+        ax5.set(xticks=[])
+        ax5.legend()
+        ax5.label_outer()
+        st.pyplot(plt.gcf())
+
+        fig6, ax6= plt.subplots(figsize=(6, 2))            
+        y = librosa.effects.harmonic(data)
+        tonnetz = librosa.feature.tonnetz(y=y, sr=sample_rate,fmin=75)
+        img_tonnetz = librosa.display.specshow(tonnetz,
+                            y_axis='tonnetz', x_axis='time', ax=ax6)
+        ax6.set(title=f'Tonal Centroids(Tonnetz)')
+        ax6.label_outer()
+        fig6.colorbar(img_tonnetz, ax=ax6)
+        st.pyplot(plt.gcf())
+
+                        
+                        
+def find_definition(emo):
+    word_to_search = emo
+    scrape_url = 'https://www.oxfordlearnersdictionaries.com/definition/english/'  + word_to_search
+
+    headers = {"User-Agent": "mekmek"}
+    web_response = requests.get(scrape_url, headers=headers)
+    
+    if web_response.status_code == 200:
+        soup = BeautifulSoup(web_response.text, 'html.parser')
+
+        try:
+            for sense in soup.find_all('li', class_='sense'):
+                definition = sense.find('span', class_='def').text
+            for example in soup.find_all('ul', class_='examples'):
+                example_1 = example.text.split('.')[0:1]
+        except AttributeError:
+            print('Word not found!!')
+    else:
+        print('Failed to get response...')
+    return definition, example_1
+                           
+def get_search_results(emo):
+    results_lis =[]
+    results = search(f"Understanding {emo}", num_results=3)
+    for result in results:
+        results_lis.append(result)
+    result_1 = results_lis[0]
+    result_2 = results_lis[1]
+    result_3 = results_lis[2] 
+    return result_1, result_2, result_3
+   
+def get_content(emo):
+        definition, example_1 = find_definition(emo)
+        result_1, result_2, result_3 = get_search_results(emo)
+
+
+        with st.expander(f"Word Definition of {emo.capitalize()}"):
+            st.write(definition.capitalize()+'.')
+        with st.expander(f'Example of {emo.capitalize()}'): 
+            with st.container():
+                st.write(f'1) {example_1[0].capitalize()}'+'.')
+
+        with st.expander(f'The following links will help you understand more on {emo.capitalize()}'): 
+            with st.container():
+                st.write(f"Check out this link ➡ {result_1}")
+                st.write(f"Check out this link ➡ {result_2}")
+                st.write(f"Check out this link ➡ {result_3}") 
+
+        if emo == 'anger':
+            with st.expander(f'Video on {emo.capitalize()}'): 
+                with st.container():
+                    st.video('https://www.youtube.com/watch?v=weMeIh10cLs')
+        if emo in 'happiness':
+            with st.expander(f'Video on {emo.capitalize()}'): 
+                with st.container():
+                    st.video('https://www.youtube.com/watch?v=FDF2DidUAyY')                        
+        if emo in 'sadness':
+            with st.expander(f'Video on {emo.capitalize()}'): 
+                with st.container():
+                    st.video('https://www.youtube.com/watch?v=34rqQEkuhK4')                        
+        if emo in 'surprised':
+            with st.expander(f'Video on {emo.capitalize()}'): 
+                with st.container():
+                    st.video('https://www.youtube.com/watch?v=UYoBi0EssLE')            
+
+        
+def upload():
+    upload_file = st.sidebar.file_uploader('Upload an audio .wav file. Currently max 8 seconds', type=".wav", accept_multiple_files = False)
+
+    if upload_file:
+        st.write('Sample Audio')
+        st.audio(upload_file, format='audio/wav')
+
+        if st.sidebar.button('Show Features'):
+            with st.spinner(f'Showing....'):
+                data_ori, sample_rate = librosa.load(upload_file)
+                data, _ = librosa.effects.trim(data_ori)
+                
+                plot_features(data,sample_rate)
+            st.sidebar.success("Completed")
+
+        if st.sidebar.button('Classify'):
+            with st.spinner(f'Classifying....'):
+                test = prepare_test(upload_file)
+                pred = classifier.predict(test)
+                pred_df = pd.DataFrame(pred.T,index=['anger', 'happiness','neutral','sadness','surprised'],columns =['Scores'])
+                emo = pred_df[pred_df['Scores'] == pred_df.max().values[0]].index[0]
+                st.info(f'The predicted Emotion: {emo.upper()}')
+            st.sidebar.success("Classification completed")
+            
+            if emo:
+                get_content(emo)
+                
+def record():
+    with st.spinner(f'Recording....'):
+        st.sidebar.write('To start press Start Recording and stop to finish recording')
+        parent_dir = os.path.dirname(os.path.abspath(__file__))
+        build_dir = os.path.join(parent_dir, "st_audiorec/frontend/build")
+        st_audiorec = components.declare_component("st_audiorec", path=build_dir)
+        val = st_audiorec()
+    
+    
+    if isinstance(val, dict):
+        st.sidebar.success("Audio Recorded")
+        ind, val = zip(*val['arr'].items())
+        ind = np.array(ind, dtype=int)
+        val = np.array(val)
+        sorted_ints = val[ind]
+        stream = BytesIO(b"".join([int(v).to_bytes(1, "big") for v in sorted_ints]))
+        wav_bytes = stream.read()
+        rate, data = read(BytesIO(wav_bytes))
+        reversed_data = data[::-1] 
+        bytes_wav = bytes()
+        byte_io = BytesIO(bytes_wav)
+        write(byte_io, rate, reversed_data)
+
+        if st.sidebar.button('Show Features'):
+            with st.spinner(f'Showing....'):
+                data_ori, sample_rate = librosa.load(byte_io)
+                data, _ = librosa.effects.trim(data_ori)
+
+                plot_features(data,sample_rate)
+            st.sidebar.success("Completed")
+
+        if st.sidebar.button('Classify'):
+            with st.spinner(f'Classifying....'):
+                test = prepare_test(byte_io)
+                pred = classifier.predict(test)
+                pred_df = pd.DataFrame(pred.T,index=['anger', 'happiness','neutral','sadness','surprised'],columns =['Scores'])
+                emo = pred_df[pred_df['Scores'] == pred_df.max().values[0]].index[0]
+                st.info(f'The predicted Emotion: {emo.upper()}')
+            st.sidebar.success("Classification completed")
+                        
+            if emo:
+                get_content(emo)                        
+                        

.ipynb_checkpoints/requirements-checkpoint.txt

@@ -0,0 +1,15 @@
+beautifulsoup4==4.9.3
+googlesearch_python==1.1.0
+ipython==8.6.0
+keras==2.10.0
+tensorflow==2.10.0
+librosa==0.9.2
+matplotlib==3.6.0
+numpy==1.22.1
+pandas==1.4.4
+requests==2.25.1
+scikit_learn==1.1.3
+soundfile==0.11.0
+scipy==1.7.3
+streamlit==1.14.0
+helper==2.5.0

Audio_Classifier.py

@@ -1,5 +1,4 @@
 import streamlit as st
-from streamlit_chat import message
 from helper import model, upload, record
 
 title = "Emotion Audio Classifier"

helper.py

@@ -1,5 +1,4 @@
 import streamlit as st
-from streamlit_chat import message
 from keras.models import load_model
 import os
 import streamlit as st
@@ -166,7 +165,7 @@ def find_definition(emo):
                            
 def get_search_results(emo):
     results_lis =[]
-    results = search(f"Understanding {emo}", num_results=1)
+    results = search(f"Understanding {emo}", num_results=3)
     for result in results:
         results_lis.append(result)
     result_1 = results_lis[0]
@@ -195,15 +194,15 @@ def get_content(emo):
             with st.expander(f'Video on {emo.capitalize()}'): 
                 with st.container():
                     st.video('https://www.youtube.com/watch?v=weMeIh10cLs')
-        if emo in 'happy':
+        if emo in 'happiness':
             with st.expander(f'Video on {emo.capitalize()}'): 
                 with st.container():
                     st.video('https://www.youtube.com/watch?v=FDF2DidUAyY')                        
-        if emo in 'sad':
+        if emo in 'sadness':
             with st.expander(f'Video on {emo.capitalize()}'): 
                 with st.container():
                     st.video('https://www.youtube.com/watch?v=34rqQEkuhK4')                        
-        if emo in 'suprise':
+        if emo in 'surprised':
             with st.expander(f'Video on {emo.capitalize()}'): 
                 with st.container():
                     st.video('https://www.youtube.com/watch?v=UYoBi0EssLE')            
@@ -228,7 +227,7 @@ def upload():
             with st.spinner(f'Classifying....'):
                 test = prepare_test(upload_file)
                 pred = classifier.predict(test)
-                pred_df = pd.DataFrame(pred.T,index=['anger', 'happy','neutral','sad','suprise'],columns =['Scores'])
+                pred_df = pd.DataFrame(pred.T,index=['anger', 'happiness','neutral','sadness','surprised'],columns =['Scores'])
                 emo = pred_df[pred_df['Scores'] == pred_df.max().values[0]].index[0]
                 st.info(f'The predicted Emotion: {emo.upper()}')
             st.sidebar.success("Classification completed")
@@ -271,7 +270,7 @@ def record():
             with st.spinner(f'Classifying....'):
                 test = prepare_test(byte_io)
                 pred = classifier.predict(test)
-                pred_df = pd.DataFrame(pred.T,index=['anger', 'happy','neutral','sad','suprise'],columns =['Scores'])
+                pred_df = pd.DataFrame(pred.T,index=['anger', 'happiness','neutral','sadness','surprised'],columns =['Scores'])
                 emo = pred_df[pred_df['Scores'] == pred_df.max().values[0]].index[0]
                 st.info(f'The predicted Emotion: {emo.upper()}')
             st.sidebar.success("Classification completed")

pages/.ipynb_checkpoints/Classify by file upload-checkpoint.py

@@ -1,5 +1,4 @@
 import streamlit as st
-from streamlit_chat import message
 from helper import model, upload, record
 
 title = "Emotion Audio Classifier with uploaded file"

pages/.ipynb_checkpoints/Classify by recordings-checkpoint.py

@@ -1,5 +1,4 @@
 import streamlit as st
-from streamlit_chat import message
 from helper import model, upload, record
 
 title = "Emotion Audio Classifier with recordings"

pages/Classify by file upload.py

@@ -1,5 +1,4 @@
 import streamlit as st
-from streamlit_chat import message
 from helper import model, upload, record
 
 title = "Emotion Audio Classifier with uploaded file"

pages/Classify by recordings.py

@@ -1,5 +1,4 @@
 import streamlit as st
-from streamlit_chat import message
 from helper import model, upload, record
 
 title = "Emotion Audio Classifier with recordings"

requirements.txt

@@ -12,5 +12,4 @@ scikit_learn==1.1.3
 soundfile==0.11.0
 scipy==1.7.3
 streamlit==1.14.0
-streamlit_chat==0.0.2.1
 helper==2.5.0