Update app.py

app.py

@@ -5,23 +5,19 @@ import wavio
 import numpy as np
 from tqdm import tqdm
 from huggingface_hub import snapshot_download
-
 from audioldm.audio.stft import TacotronSTFT
 from audioldm.variational_autoencoder import AutoencoderKL
-
 from transformers import AutoTokenizer, T5ForConditionalGeneration
 from modelling_deberta_v2 import DebertaV2ForTokenClassificationRegression
-
 import sys
-sys.path.insert(0, "diffusers/src")
 
+sys.path.insert(0, "diffusers/src")
 from diffusers import DDPMScheduler
 from models import MusicAudioDiffusion
-
 from gradio import Markdown
-
 import spaces
 
+
 # Automatic device detection
 if torch.cuda.is_available():
     device_type = "cuda"
@@ -29,7 +25,8 @@ if torch.cuda.is_available():
 else:
     device_type = "cpu"
     device_selection = "cpu"
-    
+
+
 class MusicFeaturePredictor:
     def __init__(self, path, device=device_selection, cache_dir=None, local_files_only=False):
         self.beats_tokenizer = AutoTokenizer.from_pretrained(
@@ -69,7 +66,11 @@ class MusicFeaturePredictor:
 
     def generate_beats(self, prompt):
         tokenized = self.beats_tokenizer(
-            prompt, max_length=512, padding=True, truncation=True, return_tensors="pt"
+            prompt,
+            max_length=512,
+            padding=True,
+            truncation=True,
+            return_tensors="pt",
         )
         tokenized = {k: v.to(self.beats_model.device) for k, v in tokenized.items()}
 
@@ -79,6 +80,7 @@ class MusicFeaturePredictor:
         max_beat = (
             1 + torch.argmax(out["logits"][:, 0, :], -1).detach().cpu().numpy()
         ).tolist()[0]
+
         intervals = (
             out["values"][:, :, 0]
             .detach()
@@ -88,14 +90,15 @@ class MusicFeaturePredictor:
             .round(4)
             .tolist()
         )
-
         intervals = np.cumsum(intervals)
+
         predicted_beats_times = []
         for t in intervals:
             if t < 10:
                 predicted_beats_times.append(round(t, 2))
             else:
                 break
+
         predicted_beats_times = list(np.array(predicted_beats_times)[:50])
 
         if len(predicted_beats_times) == 0:
@@ -111,7 +114,7 @@ class MusicFeaturePredictor:
     def generate(self, prompt):
         max_beat, predicted_beats_times, predicted_beats = self.generate_beats(prompt)
 
-        chords_prompt = "Caption: {} \\n Timestamps: {} \\n Max Beat: {}".format(
+        chords_prompt = "Caption: {} \n Timestamps: {} \n Max Beat: {}".format(
             prompt,
             " , ".join([str(round(t, 2)) for t in predicted_beats_times]),
             max_beat,
@@ -162,7 +165,10 @@ class Mustango:
         path = snapshot_download(repo_id=name, cache_dir=cache_dir)
 
         self.music_model = MusicFeaturePredictor(
-            path, device, cache_dir=cache_dir, local_files_only=local_files_only
+            path,
+            device,
+            cache_dir=cache_dir,
+            local_files_only=local_files_only,
         )
 
         vae_config = json.load(open(f"{path}/configs/vae_config.json"))
@@ -170,7 +176,6 @@ class Mustango:
         main_config = json.load(open(f"{path}/configs/main_config.json"))
 
         ALT_SD21 = "sd2-community/stable-diffusion-2-1"  # <-- mets TON repo alternatif exact
-
         if main_config.get("scheduler_name") == "stabilityai/stable-diffusion-2-1":
             main_config["scheduler_name"] = ALT_SD21
 
@@ -181,16 +186,20 @@ class Mustango:
             main_config["scheduler_name"],
             unet_model_config_path=f"{path}/configs/music_diffusion_model_config.json",
         ).to(device)
+
         # self.model.device = device
 
         vae_weights = torch.load(
-            f"{path}/vae/pytorch_model_vae.bin", map_location=device
+            f"{path}/vae/pytorch_model_vae.bin",
+            map_location=device,
         )
         stft_weights = torch.load(
-            f"{path}/stft/pytorch_model_stft.bin", map_location=device
+            f"{path}/stft/pytorch_model_stft.bin",
+            map_location=device,
         )
         main_weights = torch.load(
-            f"{path}/ldm/pytorch_model_ldm.bin", map_location=device
+            f"{path}/ldm/pytorch_model_ldm.bin",
+            map_location=device,
         )
 
         self.vae.load_state_dict(vae_weights)
@@ -204,14 +213,15 @@ class Mustango:
         self.model.eval()
 
         self.scheduler = DDPMScheduler.from_pretrained(
-            main_config["scheduler_name"], subfolder="scheduler"
+            main_config["scheduler_name"],
+            subfolder="scheduler",
         )
 
     def generate(self, prompt, steps=100, guidance=3, samples=1, disable_progress=True):
         """Genrate music for a single prompt string."""
-
         with torch.no_grad():
             beats, chords, chords_times = self.music_model.generate(prompt)
+
             latents = self.model.inference(
                 [prompt],
                 beats,
@@ -223,10 +233,11 @@ class Mustango:
                 samples,
                 disable_progress,
             )
+
             mel = self.vae.decode_first_stage(latents)
             wave = self.vae.decode_to_waveform(mel)
 
-        return wave[0]
+            return wave[0]
 
 
 # Initialize Mustango
@@ -236,36 +247,52 @@ mustango.stft.to(device_type)
 mustango.model.to(device_type)
 mustango.music_model.beats_model.to(device_type)
 mustango.music_model.chords_model.to(device_type)
-        
+
 # if torch.cuda.is_available():
 #     mustango = Mustango(device=device_selection)
 # else:
 #     mustango = Mustango(device="CPU")
 
 # mustango = Mustango(device=device_selection)
-
 mustango.model.device = device_selection
 
-    
 # output_wave = mustango.generate("This techno song features a synth lead playing the main melody.", 5, 3, disable_progress=False)
+
+
 @spaces.GPU(duration=120)
-def gradio_generate(prompt, steps, guidance):
-    output_wave = mustango.generate(prompt, steps, guidance)
+def generate_music_from_prompt(music_prompt: str, denoising_steps: int, guidance_scale: float) -> str:
+    """
+    Generate a short music audio file from a text prompt using Mustango.
+
+    Use this tool when a user wants to create controllable text-to-music audio from a detailed music description.
+
+    Args:
+        music_prompt (str): Text description of the desired music, including instruments, mood, tempo, key, chords, or scene.
+        denoising_steps (int): Number of diffusion denoising steps to use for generation.
+        guidance_scale (float): Classifier-free guidance scale controlling prompt adherence.
+
+    Returns:
+        str: Filepath to the generated WAV audio file.
+    """
+    output_wave = mustango.generate(music_prompt, denoising_steps, guidance_scale)
     # output_filename = f"{prompt.replace(' ', '_')}_{steps}_{guidance}"[:250] + ".wav"
     output_filename = "temp.wav"
     wavio.write(output_filename, output_wave, rate=16000, sampwidth=2)
-    
     return output_filename
 
 
-title="Mustango: Toward Controllable Text-to-Music Generation"
+title = "Mustango: Toward Controllable Text-to-Music Generation"
+
 description_text = """
-<p><a href="https://huggingface.co/spaces/declare-lab/mustango/blob/main/app.py?duplicate=true"> <img style="margin-top: 0em; margin-bottom: 0em" src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a> For faster inference without waiting in queue, you may duplicate the space and upgrade to a GPU in the settings. <br/><br/>
+For faster inference without waiting in queue, you may duplicate the space and upgrade to a GPU in the settings.
+
 Generate music using Mustango by providing a text prompt.
-<br/><br/> This is the demo for Mustango for controllable text to music generation: <a href="https://arxiv.org/abs/2311.08355">Read our paper.</a>
-<p/>
+
+This is the demo for Mustango for controllable text to music generation: [Read our paper.](https://arxiv.org/abs/2311.08355)
 """
-#description_text = ""
+
+# description_text = ""
+
 # Gradio input and output components
 input_text = gr.Textbox(lines=2, label="Prompt")
 output_audio = gr.Audio(label="Generated Music", type="filepath")
@@ -273,37 +300,69 @@ denoising_steps = gr.Slider(minimum=100, maximum=200, value=100, step=1, label="
 guidance_scale = gr.Slider(minimum=1, maximum=10, value=3, step=0.1, label="Guidance Scale", interactive=True)
 
 # CSS styling for the Duplicate button
-css = '''
+css = """
 #duplicate-button {
-  margin: auto;
-  color: white;
-  background: #1565c0;
-  border-radius: 100vh;
+    margin: auto;
+    color: white;
+    background: #1565c0;
+    border-radius: 100vh;
 }
-'''
+"""
 
 # Gradio interface
 gr_interface = gr.Interface(
-    fn=gradio_generate,
+    fn=generate_music_from_prompt,
     inputs=[input_text, denoising_steps, guidance_scale],
     outputs=[output_audio],
     description=description_text,
     examples=[
-        ["This techno song features a synth lead playing the main melody. This is accompanied by programmed percussion playing a simple kick focused beat. The hi-hat is accented in an open position on the 3-and count of every bar. The synth plays the bass part with a voicing that sounds like a cello. This techno song can be played in a club. The chord sequence is Gm, A7, Eb, Bb, C, F, Gm. The beat counts to 2. The tempo of this song is 128.0 beats per minute. The key of this song is G minor.", 200, 3],
-        ["This is a new age piece. There is a flute playing the main melody with a lot of staccato notes. The rhythmic background consists of a medium tempo electronic drum beat with percussive elements all over the spectrum. There is a playful atmosphere to the piece. This piece can be used in the soundtrack of a children's TV show or an advertisement jingle.", 200, 3],
-        ["The song is an instrumental. The song is in medium tempo with a classical guitar playing a lilting melody in accompaniment style. The song is emotional and romantic. The song is a romantic instrumental song. The chord sequence is Gm, F6, Ebm. The time signature is 4/4. This song is in Adagio. The key of this song is G minor.", 200, 3],
-        ["This folk song features a female voice singing the main melody. This is accompanied by a tabla playing the percussion. A guitar strums chords. For most parts of the song, only one chord is played. At the last bar, a different chord is played. This song has minimal instruments. This song has a story-telling mood. This song can be played in a village scene in an Indian movie. The chord sequence is Bbm, Ab. The beat is 3. The tempo of this song is Allegro. The key of this song is Bb minor.", 200, 3],
-        ["This is a live performance of a classical music piece. There is an orchestra performing the piece with a violin lead playing the main melody. The atmosphere is sentimental and heart-touching. This piece could be playing in the background at a classy restaurant. The chord progression in this song is Am7, Gm, Dm, A7, Dm. The beat is 3. This song is in Largo. The key of this song is D minor.", 200, 3],
-        ["This is a techno piece with drums and beats and a leading melody. A synth plays chords. The music kicks off with a powerful and relentless drumbeat. Over the pounding beats, a leading melody emerges. In the middle of the song, a flock of seagulls flies over the venue and make loud bird sounds. It has strong danceability and can be played in a club. The tempo is 120 bpm. The chords played by the synth are Am, Cm, Dm, Gm.", 200, 3],
+        [
+            """This techno song features a synth lead playing the main melody.
+This is accompanied by programmed percussion playing a simple kick focused beat. The hi-hat is accented in an open position on the 3-and count of every bar. The synth plays the bass part with a voicing that sounds like a cello. This techno song can be played in a club. The chord sequence is Gm, A7, Eb, Bb, C, F, Gm. The beat counts to 2. The tempo of this song is 128.0 beats per minute. The key of this song is G minor.""",
+            200,
+            3,
+        ],
+        [
+            """This is a new age piece.
+There is a flute playing the main melody with a lot of staccato notes. The rhythmic background consists of a medium tempo electronic drum beat with percussive elements all over the spectrum. There is a playful atmosphere to the piece. This piece can be used in the soundtrack of a children's TV show or an advertisement jingle.""",
+            200,
+            3,
+        ],
+        [
+            """The song is an instrumental. The song is in medium tempo with a classical guitar playing a lilting melody in accompaniment style. The song is emotional and romantic.
+The song is a romantic instrumental song. The chord sequence is Gm, F6, Ebm. The time signature is 4/4. This song is in Adagio. The key of this song is G minor.""",
+            200,
+            3,
+        ],
+        [
+            """This folk song features a female voice singing the main melody. This is accompanied by a tabla playing the percussion. A guitar strums chords. For most parts of the song, only one chord is played. At the last bar, a different chord is played. This song has minimal instruments. This song has a story-telling mood.
+This song can be played in a village scene in an Indian movie. The chord sequence is Bbm, Ab. The beat is 3. The tempo of this song is Allegro. The key of this song is Bb minor.""",
+            200,
+            3,
+        ],
+        [
+            """This is a live performance of a classical music piece. There is an orchestra performing the piece with a violin lead playing the main melody. The atmosphere is sentimental and heart-touching. This piece could be playing in the background at a classy restaurant. The chord progression in this song is Am7, Gm, Dm, A7, Dm.
+The beat is 3. This song is in Largo. The key of this song is D minor.""",
+            200,
+            3,
+        ],
+        [
+            """This is a techno piece with drums and beats and a leading melody. A synth plays chords. The music kicks off with a powerful and relentless drumbeat. Over the pounding beats, a leading melody emerges. In the middle of the song, a flock of seagulls flies over the venue and make loud bird sounds. It has strong danceability and can be played in a club. The tempo is 120 bpm.
+The chords played by the synth are Am, Cm, Dm, Gm.""",
+            200,
+            3,
+        ],
     ],
     cache_examples=False,
 )
 
-with gr.Blocks(css=css) as demo:
-    title=gr.HTML(f"<h1><center>{title}</center></h1>")
+with gr.Blocks() as demo:
+    title = gr.HTML(f"""
+# {title}
+""")
     dupe = gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
     gr_interface.render()
-   
+
 
 # Launch Gradio app.
-demo.queue().launch(ssr_mode=False)
+demo.queue().launch(css=css, ssr_mode=False, mcp_server=True)