support pitch contour and db plotting

app.py

@@ -8,7 +8,7 @@ import pdb
 import jiwer
 from local.convert_metrics import nat2avaMOS, WER2INTELI
 from local.indicator_plot import Intelligibility_Plot, Naturalness_Plot
-
+from local.pitch_contour import draw_spec_db_pitch
 # ASR part
 from transformers import pipeline
 p = pipeline("automatic-speech-recognition")
@@ -85,77 +85,21 @@ def calc_mos(audio_path, ref):
     phone_transcription = processor.batch_decode(phone_predicted_ids)
     lst_phonemes = phone_transcription[0].split(" ")
     wav_vad = torchaudio.functional.vad(wav, sample_rate=sr)
-    import matplotlib.pyplot as plt
-    
-    fig = plt.figure(figsize=(30, 10))
-    # ax = fig.subplots(1, 1)
-    # pdb.set_trace()
-    
-    # time_x = torch.arange(wav.shape[-1]) / sr
-    # # ax.plot(time_x, wav_vad.squeeze())
-    # pdb.set_trace()
-    # ax.plot(time_x, wav.squeeze(), alpha=0.5)
-    # get f0
-    f0 = torchaudio.functional.compute_kaldi_pitch(wav, frame_length=25, frame_shift=20, min_f0=20, max_f0=600, sample_rate=sr)[0, :, 1]
-    # # get f0 time x axis
-    # time_x_f0 = torch.arange(f0.shape[-1]) * 20 / 1000
-    # plot f0 with x axis as time
-    
-    # spectrogram with x axis as time
-    # pdb.set_trace()
-    spectrogram = torchaudio.transforms.MelSpectrogram(sample_rate=sr, n_fft=400, hop_length=160, n_mels=80)(wav)
-    
-    spectrogram = torchaudio.transforms.AmplitudeToDB(stype="power", top_db=80)(spectrogram)
-    
-    # plot spectrogram with x axis as time, y axis as frequency bins
-    ax2 = fig.add_subplot(212)
-    ax2.set_xlabel("Time (s)")
-    ax2.set_ylabel("Frequency (Hz)")
-    ax2.set_title("Spectrogram")
-    ax2.set_xticks(torch.arange(0, spectrogram.shape[-1], 100))
-    ax2.set_xticklabels(torch.arange(0, spectrogram.shape[-1], 100) * 20 / 1000)
-    ax2.set_yticks(torch.arange(0, spectrogram.shape[1], 10))
-    ax2.set_yticklabels(torch.arange(0, spectrogram.shape[1], 10) * 800 / 80)
-    
-    # add colorbar to spectrogram with limitation from -80 to 0
-    cbar = plt.colorbar(ax2.imshow(spectrogram.squeeze().numpy(), aspect='auto', origin='lower'))
-    cbar.set_label("dB")
-    ax2.grid()
-    
-    # plot f0 with x axis as time, y axis as frequency bins, y is limited from 0 to 600
-    ax1 = fig.add_subplot(211)
-    ax1.set_xlabel("Time (s)")
-    ax1.set_ylabel("Frequency (Hz)")
-    ax1.set_title("F0")
-    ax1.set_xticks(torch.arange(0, f0.shape[-1], 100))
-    ax1.set_xticklabels(torch.arange(0, f0.shape[-1], 100) * 20 / 1000)
-    ax1.set_yticks(torch.arange(0, 600, 50))
-    ax1.set_yticklabels(torch.arange(0, 600, 50))
 
-    # add colorbar to f0 with limitation from 0 to 600
-    # cbar = plt.colorbar(ax1.imshow(f0.squeeze().numpy(), aspect='auto', origin='lower'))
-    # cbar.set_label("Hz")
-    ax1.grid()
-    
-    # remove unvoiced part based on vad
-    
-    # plot f0 with x axis as time
-    
-    # time_x = torch.arange(f0.shape[-1]) * 20 / 1000
-    # plt.plot(time_x, f0.squeeze())
-    # fig.savefig("vad.png")
-    # pdb.set_trace()
+    # draw f0 and db analysis plot
+    f0_db_fig = draw_spec_db_pitch(audio_path, save_fig_path=None)
+
     ppm = len(lst_phonemes) / (wav_vad.shape[-1] / sr) * 60
     
     # pdb.set_trace()
-    return AVA_MOS, MOS_fig, INTELI_score, INT_fig, trans, phone_transcription, ppm
+    return AVA_MOS, MOS_fig, INTELI_score, INT_fig, trans, phone_transcription, ppm , f0_db_fig
 
 
 with open("local/description.md") as f:
     description = f.read()
 
-# calc_mos("JOHN1.wav", "he would answer in a soft voice, 'I don't know.'")
-
+# x = calc_mos("JOHN1.wav", "he would answer in a soft voice, 'I don't know.'")
+# pdb.set_trace()
 
 examples = [
     ["local/Julianna_Set1_Author_01.wav", "Once upon a time, there was a young rat named Arthur who couldn't make up his mind."],
@@ -171,11 +115,12 @@ iface = gr.Interface(
            gr.Plot(label="Intelligibility Score, range from 0 to 100, the higher the better", show_label=True, container=True),
            gr.Textbox(placeholder="Hypothesis", label="Hypothesis"),
            gr.Textbox(placeholder="Predicted Phonemes", label="Predicted Phonemes", visible=False),
-           gr.Textbox(placeholder="Speaking Rate, Phonemes per minutes", label="Speaking Rate, Phonemes per minutes", visible=False)],
+           gr.Textbox(placeholder="Speaking Rate, Phonemes per minutes", label="Speaking Rate, Phonemes per minutes", visible=False),
+           gr.Plot(label="Pitch Contour and dB Analysis", show_label=True, container=True)],
   title="Speech Analysis by Laronix AI",
   description=description,
   allow_flagging="auto",
   examples=examples,
 )
 # add password to protect the interface
-iface.launch(share=False, auth=['Laronix', 'LaronixSLP'], auth_message="Authentication Required, ask kevin@laronix.com for password,\n Thanks for your cooperation!")
+iface.launch(share=False, auth=['Laronix', 'LaronixSLP'], auth_message="Authentication Required, ask kevin@laronix.com for password.\n Thanks for your cooperation!")

local/pitch_contour.py

@@ -0,0 +1,107 @@
+### Kevin @ Laronix
+
+from glob import glob
+from pathlib import Path
+import matplotlib
+from matplotlib.transforms import Bbox  
+import matplotlib.pyplot as plt
+
+import numpy as np
+import pdb
+import parselmouth
+
+def draw_spectrogram(spectrogram, dynamic_range=80):
+    X, Y = spectrogram.x_grid(), spectrogram.y_grid()
+    sg_db = 10 * np.log10(spectrogram.values)
+    plt.pcolormesh(X, Y, sg_db, vmin=sg_db.max() - dynamic_range)
+    plt.ylim([spectrogram.ymin, spectrogram.ymax])
+    # TODO add colorbar to spectrogram with limitation from -40 to 0
+    
+    # plt.xlabel("time [s]")
+    plt.ylabel("frequency [Hz]")
+
+def draw_intensity(intensity):
+    # draw intensity in red with x axis as time
+    plt.plot(intensity.xs(), intensity.values.T, linewidth=3, color='r')
+    plt.plot(intensity.xs(), intensity.values.T, linewidth=1, color="w")
+    intensity_values = intensity.values.T
+    # get range of intensity
+    intensity_min = np.nanmin(intensity_values)
+    intensity_max = np.nanmax(intensity_values)
+    # project maximum and minimum intensity to y axis in dotted line
+    intensity_min_index = np.where(intensity.values.T == intensity_min)
+    intensity_min_time = intensity.xs()[intensity_min_index[0]][0]
+    intensity_max_index = np.where(intensity.values.T == intensity_max)
+    intensity_max_time = intensity.xs()[intensity_max_index[0]][0]
+    
+    plt.plot([intensity.xmax, intensity_min_time], [intensity_min, intensity_min], linewidth=1, linestyle='dotted', color='red')
+    plt.plot([intensity.xmax, intensity_max_time], [intensity_max, intensity_max], linewidth=1, linestyle='dotted', color='red')
+    # add text at intensity_min and intensity_max on y axis
+    plt.text(intensity.xmax, intensity_min, str(round(intensity_min, 1)), color='red')
+    plt.text(intensity.xmax, intensity_max, str(round(intensity_max, 1)), color='red')
+    # get intensity.min's index
+    
+    plt.grid(False)
+    plt.ylim(0)
+    plt.ylabel("intensity [dB]")
+
+def draw_pitch(pitch):
+    # Extract selected pitch contour, and
+    # replace unvoiced samples by NaN to not plot
+    pitch_values = pitch.selected_array['frequency']
+    pitch_values[pitch_values==0] = np.nan
+    pitch_min = np.nanmin(pitch_values)
+    pitch_max = np.nanmax(pitch_values)
+    plt.plot(pitch.xs(), pitch_values,  markersize=5, color='blue')
+    # plt.plot(pitch.xs(), pitch_values,  markersize=, color='white')
+    # project maximum and minimum db to y axis in dotted line
+    pitch_min_index = np.where(pitch_values == pitch_min)
+    pitch_min_time = pitch.xs()[pitch_min_index[0]][0]
+    pitch_max_index = np.where(pitch_values == pitch_max)
+    pitch_max_time = pitch.xs()[pitch_max_index[0]][0]
+    plt.plot([pitch.xmin, pitch_min_time], [pitch_min, pitch_min], linewidth=1, linestyle='dotted', color='blue')
+    plt.plot([pitch.xmin, pitch_max_time], [pitch_max, pitch_max], linewidth=1, linestyle='dotted', color='blue')
+    # add text at pitch_min and pitch_max on y axis
+    # highlight pitch_min and pitch_max
+    plt.scatter(pitch_min_time, pitch_min, color='blue', s=100)
+    plt.scatter(pitch_max_time, pitch_max, color='blue', s=100)
+    
+    plt.text(pitch_min_time-0.2, pitch_min-30, "f0min = " + str(round(pitch_min, 1) ), color='blue', fontsize=12)
+    plt.text(pitch_max_time-0.2, pitch_max+30, "f0max = " + str(round(pitch_max, 1) ), color='blue', fontsize=12)
+    
+    plt.grid(False)
+    
+    plt.ylim(max([0, pitch_min-50]), pitch_max+50)
+    plt.ylabel("fundamental frequency [Hz]")
+
+def draw_spec_db_pitch(wav, save_fig_path=None):
+    # get figure
+    fig = plt.figure(figsize=(10, 5))
+    fig.tight_layout()
+
+    # get pitch, intensity, spectrogram
+    snd = parselmouth.Sound(str(wav))
+    pitch = snd.to_pitch()
+    intensity = snd.to_intensity()
+    pre_emphasized_snd = snd.copy()
+    pre_emphasized_snd.pre_emphasize()
+    spectrogram = pre_emphasized_snd.to_spectrogram(window_length=0.1)
+
+    # draw dB plot and spectrogram
+    plt.subplot(2, 1, 1)
+    draw_spectrogram(spectrogram)
+    plt.twinx()
+    draw_intensity(intensity)
+    plt.xlim([snd.xmin, snd.xmax])
+
+    # draw pitch contour
+    plt.subplot(2, 1, 2)
+    draw_pitch(pitch)
+    plt.xlim([snd.xmin, snd.xmax])
+    plt.xlabel("time [s]")
+    
+    return fig
+
+# f = draw_spec_db_pitch("./test.wav")
+# plt.savefig("y.png")
+