Melody

app.py

@@ -35,10 +35,13 @@ def model_interface(model_name, top_k, top_p, temperature, cfg_coef, segments, o
     )
     
     extension_parameters = {"segments":segments, "overlap":overlap}
+    optional_audio_parameters = {"optional_audio":optional_audio, "sample_rate":loaded_model.sample_rate}
     
     prompts = split_prompt(prompt, segments)
     first_prompt = prompts[0]
-    sample_rate, audio = predict(loaded_model, prompts, None, extension_parameters)
+
+    sample_rate, audio = predict(loaded_model, prompts, duration, optional_audio_parameters, extension_parameters)
+    
     counter = 1
     audio_path = "static/"
     audio_name = first_prompt
@@ -82,7 +85,7 @@ with gr.Blocks() as interface:
         with gr.Column():
             with gr.Row():
                 model_dropdown = gr.components.Dropdown(choices=["small", "medium", "large", "melody"], label="Model Size", value="large")
-                optional_audio = gr.components.Audio(source="upload", type="filepath", label="Optional Audio", interactive=True)
+                optional_audio = gr.components.Audio(source="upload", type="numpy", label="Optional Audio", interactive=True)
 
             slider_keys = list(slider_param.keys())
             slider_pairs = list(zip_longest(slider_keys[::2], slider_keys[1::2]))

audio.py

@@ -1,16 +1,17 @@
 import numpy as np
 import os, re, json, sys
 import torch, torchaudio, pathlib
+from audiocraft.data.audio_utils import convert_audio
 
-def load_and_process_audio(model, melody, sample_rate):
-    if melody is not None:
-        melody = torch.from_numpy(melody).to(model.device).float().t().unsqueeze(0)
-        if melody.dim() == 2:
-            melody = melody[None]
-        melody = melody[..., :int(sample_rate * model.lm.cfg.dataset.segment_duration)]
-        return melody
-    else:
+def load_and_process_audio(model, duration, optional_audio, sample_rate):
+    if optional_audio is None:
         return None
+    sr, optional_audio = optional_audio[0], torch.from_numpy(optional_audio[1]).to(model.device).float().t()
+    if optional_audio.dim() == 1:
+        optional_audio = optional_audio[None]
+    optional_audio = optional_audio[..., :int(sr * duration)]
+    optional_audio = convert_audio(optional_audio, sr, sr, 1)
+    return optional_audio
 
 #From https://colab.research.google.com/drive/154CqogsdP-D_TfSF9S2z8-BY98GN_na4?usp=sharing#scrollTo=exKxNU_Z4i5I
 #Thank you DragonForged for the link
@@ -35,12 +36,12 @@ def extend_audio(model, prompt_waveform, prompts, prompt_sr, segments=5, overlap
 
     return prompt_waveform
 
-def predict(model, prompts, melody_parameters, extension_parameters):
-    melody = None #load_and_process_audio(MODEL, **melody_parameters)
+def predict(model, prompts, duration, melody_parameters, extension_parameters):
+    melody = load_and_process_audio(model, duration, **melody_parameters)
 
     if melody is not None:
-        output = MODEL.generate_with_chroma(
-            descriptions=[prompt[0]],
+        output = model.generate_with_chroma(
+            descriptions=[prompts[0]],
             melody_wavs=melody,
             melody_sample_rate=melody_parameters['sample_rate'],
             progress=False