Spaces:
Runtime error
Runtime error
import streamlit as st | |
from keras.models import load_model | |
import os | |
import streamlit as st | |
import numpy as np | |
import pandas as pd | |
from sklearn.preprocessing import normalize | |
import matplotlib.pyplot as plt | |
import librosa | |
import librosa.display | |
import IPython.display as ipd | |
import streamlit.components.v1 as components | |
from io import BytesIO | |
from scipy.io.wavfile import read, write | |
from googlesearch import search | |
import requests | |
from bs4 import BeautifulSoup | |
classifier = load_model('./model/bestmodel.h5') | |
def prepare_test(file): | |
max_size=350 | |
features=[] | |
data_ori, sample_rate = librosa.load(file) | |
data, _ = librosa.effects.trim(data_ori) | |
spec_bw = padding(librosa.feature.spectral_bandwidth(y = data, sr= sample_rate), 20, max_size).astype('float32') | |
cent = padding(librosa.feature.spectral_centroid(y = data, sr=sample_rate), 20, max_size).astype('float32') | |
mfcc = librosa.feature.mfcc(y=data, sr=sample_rate,n_mfcc=20) | |
mfccs = padding(normalize(mfcc, axis=1), 20, max_size).astype('float32') | |
rms = padding(librosa.feature.rms(y = data),20, max_size).astype('float32') | |
y = librosa.effects.harmonic(data) | |
tonnetz = padding(librosa.feature.tonnetz(y=y, sr=sample_rate,fmin=75),20, max_size).astype('float32') | |
image = padding(librosa.feature.chroma_cens(y = data, sr=sample_rate,fmin=75), 20, max_size).astype('float32') | |
image=np.dstack((image,spec_bw)) | |
image=np.dstack((image,cent)) | |
image=np.dstack((image,mfccs)) | |
image=np.dstack((image,rms)) | |
image=np.dstack((image,tonnetz)) | |
features.append(image[np.newaxis,...]) | |
output = np.concatenate(features,axis=0) | |
return output | |
def padding(array, xx, yy): | |
h = array.shape[0] | |
w = array.shape[1] | |
a = max((xx - h) // 2,0) | |
aa = max(0,xx - a - h) | |
b = max(0,(yy - w) // 2) | |
bb = max(yy - b - w,0) | |
return np.pad(array, pad_width=((a, aa), (b, bb)), mode='constant') | |
def model(): | |
select = st.selectbox('Please select one to find out more about the model', | |
('Choose','Model Training History', 'Confusion Matrix', 'Train-Test Scores')) | |
if select == 'Model Training History': | |
st.header('Train-Val Accuracy History') | |
st.write('Train and Validation accuracy scores are comparable this indicates that the Model is moderately trained.') | |
st.image('./img/bestmodelacc.png') | |
st.header('Train-Val Loss History') | |
st.write('The training was stopped with EarlyStopping() when the Validation loss score starts to saturate.') | |
st.image('./img/bestmodelloss.png') | |
if select == 'Confusion Matrix': | |
st.header('Confusion Matrix') | |
st.write('Below is the Confusion Matrix for this Model normalized by rows indicating Recall scores which are over 80%.') | |
st.image('./img/bestmodelcm.png') | |
if select == 'Train-Test Scores': | |
st.header('Train-Test Accuracy scores') | |
st.write('Train accuracy score is 94% and Test accuracy score is 90%. There is a 4% difference indicating that the model fitted moderately') | |
acc = [0.943, 0.908] | |
loss = [0.177, 0.304] | |
traintest_df = pd.DataFrame(list(zip(acc,loss)),columns=['Accuracy', 'loss'],index =['Train','Test']) | |
st.dataframe(traintest_df.style.format("{:.3}")) | |
st.header('Model Precision, Recall, F1-score & MCC') | |
st.write('Macro avg F1 score is 91%.') | |
st.write('Matthew’s correlation coefficient: 0.885') | |
prec = [0.92, 0.86, 0.91, 0.97, 0.88,0.91] | |
re = [0.92, 0.83, 0.96, 0.92, 0.90, 0.91] | |
f1 = [0.92, 0.85, 0.94, 0.94, 0.89, 0.91] | |
traintest_df2 = pd.DataFrame(list(zip(prec,re,f1)),columns=['Precision', 'Recall', 'F1-score'],index =['angry', 'happy', 'neutral', 'sad', 'surprise','macro avg']) | |
st.dataframe(traintest_df2.style.format("{:.3}")) | |
def plot_features(data,sample_rate): | |
fig1, ax1 = plt.subplots(figsize=(6, 2)) | |
img = librosa.display.waveshow(y = data, sr=sample_rate, x_axis="time") | |
ax1.set(title = 'Sample Waveform') | |
st.pyplot(plt.gcf()) | |
fig2, ax2 = plt.subplots(figsize=(6, 2)) | |
cens = librosa.feature.chroma_cens(y = data, sr=sample_rate,fmin=75) | |
img_cens = librosa.display.specshow(cens, y_axis = 'chroma', x_axis='time', ax=ax2) | |
ax2.set(title = 'Chroma_CENS') | |
st.pyplot(plt.gcf()) | |
fig3, ax3 = plt.subplots(figsize=(6, 2)) | |
spec_bw = librosa.feature.spectral_bandwidth(y = data, sr= sample_rate) | |
cent = librosa.feature.spectral_centroid(y = data, sr=sample_rate) | |
times = librosa.times_like(spec_bw) | |
S, phase = librosa.magphase(librosa.stft(y=data)) | |
librosa.display.specshow(librosa.amplitude_to_db(S, ref=np.max), y_axis='log', x_axis='time', ax=ax3) | |
ax3.fill_between(times, np.maximum(0, cent[0] - spec_bw[0]), | |
np.minimum(cent[0] + spec_bw[0], sample_rate/2), | |
alpha=0.5, label='Centroid +- bandwidth') | |
ax3.plot(times, cent[0], label='Spectral centroid', color='w') | |
ax3.legend(loc='lower right') | |
ax3.set(title='log Power spectrogram') | |
st.pyplot(plt.gcf()) | |
fig4, ax4 = plt.subplots(figsize=(6, 2)) | |
mfcc = librosa.feature.mfcc(y=data, sr=sample_rate,n_mfcc=40) | |
mfccs = normalize(mfcc, axis=1) | |
img_mfcc = librosa.display.specshow(mfccs, y_axis = 'mel', x_axis='time', ax=ax4) | |
ax4.set(title = 'Sample Mel-Frequency Cepstral Coefficients') | |
st.pyplot(plt.gcf()) | |
fig5, ax5 = plt.subplots(figsize=(6, 2)) | |
rms = librosa.feature.rms(y=data) | |
times = librosa.times_like(rms) | |
ax5.semilogy(times, rms[0], label='RMS Energy') | |
ax5.set_title(f'RMS Energy') | |
ax5.set(xticks=[]) | |
ax5.legend() | |
ax5.label_outer() | |
st.pyplot(plt.gcf()) | |
fig6, ax6= plt.subplots(figsize=(6, 2)) | |
y = librosa.effects.harmonic(data) | |
tonnetz = librosa.feature.tonnetz(y=y, sr=sample_rate,fmin=75) | |
img_tonnetz = librosa.display.specshow(tonnetz, | |
y_axis='tonnetz', x_axis='time', ax=ax6) | |
ax6.set(title=f'Tonal Centroids(Tonnetz)') | |
ax6.label_outer() | |
fig6.colorbar(img_tonnetz, ax=ax6) | |
st.pyplot(plt.gcf()) | |
def find_definition(emo): | |
word_to_search = emo | |
scrape_url = 'https://www.oxfordlearnersdictionaries.com/definition/english/' + word_to_search | |
headers = {"User-Agent": "mekmek"} | |
web_response = requests.get(scrape_url, headers=headers) | |
if web_response.status_code == 200: | |
soup = BeautifulSoup(web_response.text, 'html.parser') | |
try: | |
for sense in soup.find_all('li', class_='sense'): | |
definition = sense.find('span', class_='def').text | |
for example in soup.find_all('ul', class_='examples'): | |
example_1 = example.text.split('.')[0:1] | |
except AttributeError: | |
print('Word not found!!') | |
else: | |
print('Failed to get response...') | |
return definition, example_1 | |
def get_search_results(emo): | |
results_lis =[] | |
results = search(f"Understanding {emo}", num_results=3) | |
for result in results: | |
results_lis.append(result) | |
result_1 = results_lis[0] | |
result_2 = results_lis[1] | |
result_3 = results_lis[2] | |
return result_1, result_2, result_3 | |
def get_content(emo): | |
definition, example_1 = find_definition(emo) | |
result_1, result_2, result_3 = get_search_results(emo) | |
with st.expander(f"Word Definition of {emo.capitalize()}"): | |
st.write(definition.capitalize()+'.') | |
with st.expander(f'Example of {emo.capitalize()}'): | |
with st.container(): | |
st.write(f'1) {example_1[0].capitalize()}'+'.') | |
with st.expander(f'The following links will help you understand more on {emo.capitalize()}'): | |
with st.container(): | |
st.write(f"Check out this link ➡ {result_1}") | |
st.write(f"Check out this link ➡ {result_2}") | |
st.write(f"Check out this link ➡ {result_3}") | |
if emo == 'anger': | |
with st.expander(f'Video on {emo.capitalize()}'): | |
with st.container(): | |
st.video('https://www.youtube.com/watch?v=weMeIh10cLs') | |
if emo in 'happiness': | |
with st.expander(f'Video on {emo.capitalize()}'): | |
with st.container(): | |
st.video('https://www.youtube.com/watch?v=FDF2DidUAyY') | |
if emo in 'sadness': | |
with st.expander(f'Video on {emo.capitalize()}'): | |
with st.container(): | |
st.video('https://www.youtube.com/watch?v=34rqQEkuhK4') | |
if emo in 'surprised': | |
with st.expander(f'Video on {emo.capitalize()}'): | |
with st.container(): | |
st.video('https://www.youtube.com/watch?v=UYoBi0EssLE') | |
def upload(): | |
upload_file = st.sidebar.file_uploader('Upload an audio .wav file. Currently max 8 seconds', type=".wav", accept_multiple_files = False) | |
if upload_file: | |
st.write('Sample Audio') | |
st.audio(upload_file, format='audio/wav') | |
if st.sidebar.button('Show Features'): | |
with st.spinner(f'Showing....'): | |
data_ori, sample_rate = librosa.load(upload_file) | |
data, _ = librosa.effects.trim(data_ori) | |
plot_features(data,sample_rate) | |
st.sidebar.success("Completed") | |
if st.sidebar.button('Classify'): | |
with st.spinner(f'Classifying....'): | |
test = prepare_test(upload_file) | |
pred = classifier.predict(test) | |
pred_df = pd.DataFrame(pred.T,index=['anger', 'happiness','neutral','sadness','surprised'],columns =['Scores']) | |
emo = pred_df[pred_df['Scores'] == pred_df.max().values[0]].index[0] | |
st.info(f'The predicted Emotion: {emo.upper()}') | |
st.sidebar.success("Classification completed") | |
if emo: | |
get_content(emo) | |
def record(): | |
with st.spinner(f'Recording....'): | |
st.sidebar.write('To start press Start Recording and stop to finish recording') | |
parent_dir = os.path.dirname(os.path.abspath(__file__)) | |
build_dir = os.path.join(parent_dir, "st_audiorec/frontend/build") | |
st_audiorec = components.declare_component("st_audiorec", path=build_dir) | |
val = st_audiorec() | |
if isinstance(val, dict): | |
st.sidebar.success("Audio Recorded") | |
ind, val = zip(*val['arr'].items()) | |
ind = np.array(ind, dtype=int) | |
val = np.array(val) | |
sorted_ints = val[ind] | |
stream = BytesIO(b"".join([int(v).to_bytes(1, "big") for v in sorted_ints])) | |
wav_bytes = stream.read() | |
rate, data = read(BytesIO(wav_bytes)) | |
reversed_data = data[::-1] | |
bytes_wav = bytes() | |
byte_io = BytesIO(bytes_wav) | |
write(byte_io, rate, reversed_data) | |
if st.sidebar.button('Show Features'): | |
with st.spinner(f'Showing....'): | |
data_ori, sample_rate = librosa.load(byte_io) | |
data, _ = librosa.effects.trim(data_ori) | |
plot_features(data,sample_rate) | |
st.sidebar.success("Completed") | |
if st.sidebar.button('Classify'): | |
with st.spinner(f'Classifying....'): | |
test = prepare_test(byte_io) | |
pred = classifier.predict(test) | |
pred_df = pd.DataFrame(pred.T,index=['anger', 'happiness','neutral','sadness','surprised'],columns =['Scores']) | |
emo = pred_df[pred_df['Scores'] == pred_df.max().values[0]].index[0] | |
st.info(f'The predicted Emotion: {emo.upper()}') | |
st.sidebar.success("Classification completed") | |
if emo: | |
get_content(emo) | |