Update app.py

app.py

@@ -6,7 +6,7 @@
 
 # Updated to account for UI changes from https://github.com/rkfg/audiocraft/blob/long/app.py
 # also released under the MIT license.
-import numpy as np
+
 import argparse
 from concurrent.futures import ProcessPoolExecutor
 import os
@@ -82,7 +82,6 @@ def make_waveform(*args, **kwargs):
         warnings.simplefilter('ignore')
         out = gr.make_waveform(*args, **kwargs)
         print("Make a video took", time.time() - be)
-        print("Returning from make_waveform")
         return out
 
 
@@ -107,61 +106,293 @@ def _do_predictions(texts, melodies, duration, progress=False, **gen_kwargs):
             sr, melody = melody[0], torch.from_numpy(melody[1]).to(MODEL.device).float().t()
             if melody.dim() == 1:
                 melody = melody[None]
-            melody = melody.to(target_ac).to(MODEL.device).float()
-            if melody.size(0) != target_sr:
-                melody = convert_audio(melody, sr, target_sr)
-            processed_melodies.append(melody[None])
-    try:
-        outputs, infos = MODEL.generate_multiple(texts, processed_melodies, progress=progress)
-    except RuntimeError as e:
-        print(f'Runtime error in _do_predictions: {e}')
-        return []
-    print(f'Generation took {time.time() - be} seconds.')
-    return outputs, infos
-
-
-def _postprocess(output):
-    be = time.time()
-    audio_path = NamedTemporaryFile(delete=False, suffix=".mp3").name
-    file_cleaner.add(audio_path)
-    audio_write(output, audio_path)
-    print(f'Audio write took {time.time() - be} seconds.')
-    print("Returning from _postprocess")
-    return audio_path
-
-
-def _predict_single(text: str, melody: tp.Tuple[tp.Optional[int], tp.Optional[np.ndarray]], duration: float, **gen_kwargs):
-    load_model()
-    print(f'_predict_single called with text: {text}, melody: {melody}, duration: {duration}, gen_kwargs: {gen_kwargs}')
-    outputs, infos = _do_predictions([text], [melody], duration, **gen_kwargs)
-    if not outputs:
-        print("No outputs in _predict_single")
-        return None
-    output = outputs[0]
-    return _postprocess(output)
-
-
-def _predict_batch(texts: tp.List[str], melodies: tp.List[tp.Tuple[tp.Optional[int], tp.Optional[np.ndarray]]], duration: float, **gen_kwargs):
-    load_model()
-    print(f'_predict_batch called with texts: {texts}, melodies: {melodies}, duration: {duration}, gen_kwargs: {gen_kwargs}')
-    outputs, infos = _do_predictions(texts, melodies, duration, **gen_kwargs)
-    if not outputs:
-        print("No outputs in _predict_batch")
-        return [None] * len(texts)
-    return [_postprocess(output) for output in outputs]
-
-
-def launch_app():
-    launch_kwargs = dict(verbose=False, debug=False, inline=False)
-    if 'PORT' in os.environ:
-        launch_kwargs['port'] = os.environ['PORT']
-    if 'HOST' in os.environ:
-        launch_kwargs['host'] = os.environ['HOST']
-
-    # lastly launch the app with the set parameters
+            melody = melody[..., :int(sr * duration)]
+            melody = convert_audio(melody, sr, target_sr, target_ac)
+            processed_melodies.append(melody)
+
+    if any(m is not None for m in processed_melodies):
+        outputs = MODEL.generate_with_chroma(
+            descriptions=texts,
+            melody_wavs=processed_melodies,
+            melody_sample_rate=target_sr,
+            progress=progress,
+        )
+    else:
+        outputs = MODEL.generate(texts, progress=progress)
+
+    outputs = outputs.detach().cpu().float()
+    out_files = []
+    for output in outputs:
+        with NamedTemporaryFile("wb", suffix=".wav", delete=False) as file:
+            audio_write(
+                file.name, output, MODEL.sample_rate, strategy="loudness",
+                loudness_headroom_db=16, loudness_compressor=True, add_suffix=False)
+            out_files.append(pool.submit(make_waveform, file.name))
+            file_cleaner.add(file.name)
+    res = [out_file.result() for out_file in out_files]
+    for file in res:
+        file_cleaner.add(file)
+    print("batch finished", len(texts), time.time() - be)
+    print("Tempfiles currently stored: ", len(file_cleaner.files))
+    return res
+
+
+def predict_batched(texts, melodies):
+    max_text_length = 512
+    texts = [text[:max_text_length] for text in texts]
+    load_model('melody')
+    res = _do_predictions(texts, melodies, BATCHED_DURATION)
+    return [res]
+
+
+def predict_full(model, text, melody, duration, topk, topp, temperature, cfg_coef, progress=gr.Progress()):
+    global INTERRUPTING
+    INTERRUPTING = False
+    if temperature < 0:
+        raise gr.Error("Temperature must be >= 0.")
+    if topk < 0:
+        raise gr.Error("Topk must be non-negative.")
+    if topp < 0:
+        raise gr.Error("Topp must be non-negative.")
+
+    topk = int(topk)
+    load_model(model)
+
+    def _progress(generated, to_generate):
+        progress((generated, to_generate))
+        if INTERRUPTING:
+            raise gr.Error("Interrupted.")
+    MODEL.set_custom_progress_callback(_progress)
+
+    outs = _do_predictions(
+        [text], [melody], duration, progress=True,
+        top_k=topk, top_p=topp, temperature=temperature, cfg_coef=cfg_coef)
+    return outs[0]
+
+
+def toggle_audio_src(choice):
+    if choice == "mic":
+        return gr.update(source="microphone", value=None, label="Microphone")
+    else:
+        return gr.update(source="upload", value=None, label="File")
+
+
+def ui_full(launch_kwargs):
+    with gr.Blocks() as interface:
+        gr.Markdown(
+            """
+            # MusicGen
+            This is your private demo for [MusicGen](https://github.com/facebookresearch/audiocraft),
+            a simple and controllable model for music generation
+            presented at: ["Simple and Controllable Music Generation"](https://huggingface.co/papers/2306.05284)
+            """
+        )
+        with gr.Row():
+            with gr.Column():
+                with gr.Row():
+                    text = gr.Text(label="Input Text", interactive=True)
+                    with gr.Column():
+                        radio = gr.Radio(["file", "mic"], value="file",
+                                         label="Condition on a melody (optional) File or Mic")
+                        melody = gr.Audio(source="upload", type="numpy", label="File",
+                                          interactive=True, elem_id="melody-input")
+                with gr.Row():
+                    submit = gr.Button("Submit")
+                    # Adapted from https://github.com/rkfg/audiocraft/blob/long/app.py, MIT license.
+                    _ = gr.Button("Interrupt").click(fn=interrupt, queue=False)
+                with gr.Row():
+                    model = gr.Radio(["melody", "medium", "small", "large"],
+                                     label="Model", value="melody", interactive=True)
+                with gr.Row():
+                    duration = gr.Slider(minimum=1, maximum=120, value=10, label="Duration", interactive=True)
+                with gr.Row():
+                    topk = gr.Number(label="Top-k", value=250, interactive=True)
+                    topp = gr.Number(label="Top-p", value=0, interactive=True)
+                    temperature = gr.Number(label="Temperature", value=1.0, interactive=True)
+                    cfg_coef = gr.Number(label="Classifier Free Guidance", value=3.0, interactive=True)
+            with gr.Column():
+                output = gr.Video(label="Generated Music")
+        submit.click(predict_full,
+                     inputs=[model, text, melody, duration, topk, topp, temperature, cfg_coef],
+                     outputs=[output])
+        radio.change(toggle_audio_src, radio, [melody], queue=False, show_progress=False)
+        gr.Examples(
+            fn=predict_full,
+            examples=[
+                [
+                    "An 80s driving pop song with heavy drums and synth pads in the background",
+                    "./assets/bach.mp3",
+                    "melody"
+                ],
+                [
+                    "A cheerful country song with acoustic guitars",
+                    "./assets/bolero_ravel.mp3",
+                    "melody"
+                ],
+                [
+                    "90s rock song with electric guitar and heavy drums",
+                    None,
+                    "medium"
+                ],
+                [
+                    "a light and cheerly EDM track, with syncopated drums, aery pads, and strong emotions",
+                    "./assets/bach.mp3",
+                    "melody"
+                ],
+                [
+                    "lofi slow bpm electro chill with organic samples",
+                    None,
+                    "medium",
+                ],
+            ],
+            inputs=[text, melody, model],
+            outputs=[output]
+        )
+        gr.Markdown(
+            """
+            ### More details
+            The model will generate a short music extract based on the description you provided.
+            The model can generate up to 30 seconds of audio in one pass. It is now possible
+            to extend the generation by feeding back the end of the previous chunk of audio.
+            This can take a long time, and the model might lose consistency. The model might also
+            decide at arbitrary positions that the song ends.
+            **WARNING:** Choosing long durations will take a long time to generate (2min might take ~10min).
+            An overlap of 12 seconds is kept with the previously generated chunk, and 18 "new" seconds
+            are generated each time.
+            We present 4 model variations:
+            1. Melody -- a music generation model capable of generating music condition
+                on text and melody inputs. **Note**, you can also use text only.
+            2. Small -- a 300M transformer decoder conditioned on text only.
+            3. Medium -- a 1.5B transformer decoder conditioned on text only.
+            4. Large -- a 3.3B transformer decoder conditioned on text only (might OOM for the longest sequences.)
+            When using `melody`, ou can optionaly provide a reference audio from
+            which a broad melody will be extracted. The model will then try to follow both
+            the description and melody provided.
+            You can also use your own GPU or a Google Colab by following the instructions on our repo.
+            See [github.com/facebookresearch/audiocraft](https://github.com/facebookresearch/audiocraft)
+            for more details.
+            """
+        )
+
+        interface.queue().launch(**launch_kwargs)
+
+
+def ui_batched(launch_kwargs):
+    with gr.Blocks() as demo:
+        gr.Markdown(
+            """
+            # MusicGen
+            This is the demo for [MusicGen](https://github.com/facebookresearch/audiocraft),
+            a simple and controllable model for music generation
+            presented at: ["Simple and Controllable Music Generation"](https://huggingface.co/papers/2306.05284).
+            <br/>
+            <a href="https://huggingface.co/spaces/facebook/MusicGen?duplicate=true"
+                style="display: inline-block;margin-top: .5em;margin-right: .25em;" target="_blank">
+            <img style="margin-bottom: 0em;display: inline;margin-top: -.25em;"
+                src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>
+            for longer sequences, more control and no queue.</p>
+            """
+        )
+        with gr.Row():
+            with gr.Column():
+                with gr.Row():
+                    text = gr.Text(label="Describe your music", lines=2, interactive=True)
+                    with gr.Column():
+                        radio = gr.Radio(["file", "mic"], value="file",
+                                         label="Condition on a melody (optional) File or Mic")
+                        melody = gr.Audio(source="upload", type="numpy", label="File",
+                                          interactive=True, elem_id="melody-input")
+                with gr.Row():
+                    submit = gr.Button("Generate")
+            with gr.Column():
+                output = gr.Video(label="Generated Music")
+        submit.click(predict_batched, inputs=[text, melody],
+                     outputs=[output], batch=True, max_batch_size=MAX_BATCH_SIZE)
+        radio.change(toggle_audio_src, radio, [melody], queue=False, show_progress=False)
+        gr.Examples(
+            fn=predict_batched,
+            examples=[
+                [
+                    "An 80s driving pop song with heavy drums and synth pads in the background",
+                    "./assets/bach.mp3",
+                ],
+                [
+                    "A cheerful country song with acoustic guitars",
+                    "./assets/bolero_ravel.mp3",
+                ],
+                [
+                    "90s rock song with electric guitar and heavy drums",
+                    None,
+                ],
+                [
+                    "a light and cheerly EDM track, with syncopated drums, aery pads, and strong emotions bpm: 130",
+                    "./assets/bach.mp3",
+                ],
+                [
+                    "lofi slow bpm electro chill with organic samples",
+                    None,
+                ],
+            ],
+            inputs=[text, melody],
+            outputs=[output]
+        )
+        gr.Markdown("""
+        ### More details
+        The model will generate 12 seconds of audio based on the description you provided.
+        You can optionaly provide a reference audio from which a broad melody will be extracted.
+        The model will then try to follow both the description and melody provided.
+        All samples are generated with the `melody` model.
+        You can also use your own GPU or a Google Colab by following the instructions on our repo.
+        See [github.com/facebookresearch/audiocraft](https://github.com/facebookresearch/audiocraft)
+        for more details.
+        """)
+
+        demo.queue(max_size=8 * 4).launch(**launch_kwargs)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--listen',
+        type=str,
+        default='0.0.0.0' if 'SPACE_ID' in os.environ else '127.0.0.1',
+        help='IP to listen on for connections to Gradio',
+    )
+    parser.add_argument(
+        '--username', type=str, default='', help='Username for authentication'
+    )
+    parser.add_argument(
+        '--password', type=str, default='', help='Password for authentication'
+    )
+    parser.add_argument(
+        '--server_port',
+        type=int,
+        default=0,
+        help='Port to run the server listener on',
+    )
+    parser.add_argument(
+        '--inbrowser', action='store_true', help='Open in browser'
+    )
+    parser.add_argument(
+        '--share', action='store_true', help='Share the gradio UI'
+    )
+
+    args = parser.parse_args()
+
+    launch_kwargs = {}
+    launch_kwargs['server_name'] = args.listen
+
+    if args.username and args.password:
+        launch_kwargs['auth'] = (args.username, args.password)
+    if args.server_port:
+        launch_kwargs['server_port'] = args.server_port
+    if args.inbrowser:
+        launch_kwargs['inbrowser'] = args.inbrowser
+    if args.share:
+        launch_kwargs['share'] = args.share
+
+    # Show the interface
     if IS_BATCHED:
-        print("Launching batched UI.")
         ui_batched(launch_kwargs)
     else:
-        print("Launching full UI.")
-        ui_full(launch_kwargs)
+        ui_full(launch_kwargs)