|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import argparse |
|
from concurrent.futures import ProcessPoolExecutor |
|
import os |
|
from pathlib import Path |
|
import subprocess as sp |
|
from tempfile import NamedTemporaryFile |
|
import time |
|
import typing as tp |
|
import warnings |
|
|
|
import torch |
|
import gradio as gr |
|
|
|
from audiocraft.data.audio_utils import convert_audio |
|
from audiocraft.data.audio import audio_write |
|
from audiocraft.models import MusicGen |
|
|
|
harp_deps = [ |
|
"descript-audiotools"] |
|
|
|
try: |
|
import pyharp |
|
except ImportError: |
|
print("Installing harp dependencies...") |
|
sp.check_call(["pip", "install", *harp_deps]) |
|
import pyharp |
|
|
|
|
|
MODEL = None |
|
IS_BATCHED = "facebook/MusicGen" in os.environ.get('SPACE_ID', '') |
|
MAX_BATCH_SIZE = 6 |
|
BATCHED_DURATION = 15 |
|
INTERRUPTING = False |
|
|
|
_old_call = sp.call |
|
|
|
|
|
def _call_nostderr(*args, **kwargs): |
|
|
|
kwargs['stderr'] = sp.DEVNULL |
|
kwargs['stdout'] = sp.DEVNULL |
|
_old_call(*args, **kwargs) |
|
|
|
|
|
sp.call = _call_nostderr |
|
|
|
pool = ProcessPoolExecutor(3) |
|
pool.__enter__() |
|
|
|
|
|
def interrupt(): |
|
global INTERRUPTING |
|
INTERRUPTING = True |
|
|
|
|
|
class FileCleaner: |
|
def __init__(self, file_lifetime: float = 3600): |
|
self.file_lifetime = file_lifetime |
|
self.files = [] |
|
|
|
def add(self, path: tp.Union[str, Path]): |
|
self._cleanup() |
|
self.files.append((time.time(), Path(path))) |
|
|
|
def _cleanup(self): |
|
now = time.time() |
|
for time_added, path in list(self.files): |
|
if now - time_added > self.file_lifetime: |
|
if path.exists(): |
|
path.unlink() |
|
self.files.pop(0) |
|
else: |
|
break |
|
|
|
|
|
file_cleaner = FileCleaner() |
|
|
|
|
|
def make_waveform(*args, **kwargs): |
|
|
|
be = time.time() |
|
with warnings.catch_warnings(): |
|
warnings.simplefilter('ignore') |
|
out = gr.make_waveform(*args, **kwargs) |
|
print("Make a video took", time.time() - be) |
|
return out |
|
|
|
|
|
def load_model(version='melody'): |
|
global MODEL |
|
print("Loading model", version) |
|
if MODEL is None or MODEL.name != version: |
|
MODEL = MusicGen.get_pretrained(version) |
|
|
|
|
|
def _do_predictions(texts, melodies, duration, progress=False, **gen_kwargs): |
|
MODEL.set_generation_params(duration=duration, **gen_kwargs) |
|
print("new batch", len(texts), texts, [None if m is None else (m[0], m[1].shape) for m in melodies]) |
|
be = time.time() |
|
processed_melodies = [] |
|
target_sr = 32000 |
|
target_ac = 1 |
|
for melody in melodies: |
|
if melody is None: |
|
processed_melodies.append(None) |
|
else: |
|
sr, melody = melody[0], torch.from_numpy(melody[1]).to(MODEL.device).float().t() |
|
if melody.dim() == 1: |
|
melody = melody[None] |
|
melody = melody[..., :int(sr * duration)] |
|
melody = convert_audio(melody, sr, target_sr, target_ac) |
|
processed_melodies.append(melody) |
|
|
|
if any(m is not None for m in processed_melodies): |
|
outputs = MODEL.generate_with_chroma( |
|
descriptions=texts, |
|
melody_wavs=processed_melodies, |
|
melody_sample_rate=target_sr, |
|
progress=progress, |
|
) |
|
else: |
|
outputs = MODEL.generate(texts, progress=progress) |
|
|
|
outputs = outputs.detach().cpu().float() |
|
out_files = [] |
|
for output in outputs: |
|
with NamedTemporaryFile("wb", suffix=".wav", delete=False) as file: |
|
audio_write( |
|
file.name, output, MODEL.sample_rate, strategy="loudness", |
|
loudness_headroom_db=16, loudness_compressor=True, add_suffix=False) |
|
out_files.append(pool.submit(make_waveform, file.name)) |
|
file_cleaner.add(file.name) |
|
res = [out_file.result() for out_file in out_files] |
|
for file in res: |
|
file_cleaner.add(file) |
|
print("batch finished", len(texts), time.time() - be) |
|
print("Tempfiles currently stored: ", len(file_cleaner.files)) |
|
return res |
|
|
|
|
|
def predict_batched(texts, melodies): |
|
max_text_length = 512 |
|
texts = [text[:max_text_length] for text in texts] |
|
load_model('melody') |
|
res = _do_predictions(texts, melodies, BATCHED_DURATION) |
|
return [res] |
|
|
|
|
|
def predict_full(model, text, melody, duration, topk, topp, temperature, cfg_coef, progress=gr.Progress()): |
|
global INTERRUPTING |
|
INTERRUPTING = False |
|
if temperature < 0: |
|
raise gr.Error("Temperature must be >= 0.") |
|
if topk < 0: |
|
raise gr.Error("Topk must be non-negative.") |
|
if topp < 0: |
|
raise gr.Error("Topp must be non-negative.") |
|
|
|
topk = int(topk) |
|
load_model(model) |
|
|
|
def _progress(generated, to_generate): |
|
progress((generated, to_generate)) |
|
if INTERRUPTING: |
|
raise gr.Error("Interrupted.") |
|
MODEL.set_custom_progress_callback(_progress) |
|
|
|
outs = _do_predictions( |
|
[text], [melody], duration, progress=True, |
|
top_k=topk, top_p=topp, temperature=temperature, cfg_coef=cfg_coef) |
|
return outs[0] |
|
|
|
|
|
def toggle_audio_src(choice): |
|
if choice == "mic": |
|
return gr.update(source="microphone", value=None, label="Microphone") |
|
else: |
|
return gr.update(source="upload", value=None, label="File") |
|
|
|
|
|
def ui_full(launch_kwargs): |
|
with gr.Blocks() as interface: |
|
gr.Markdown( |
|
""" |
|
# MusicGen |
|
This is your private demo for [MusicGen](https://github.com/facebookresearch/audiocraft), |
|
a simple and controllable model for music generation |
|
presented at: ["Simple and Controllable Music Generation"](https://huggingface.co/papers/2306.05284) |
|
""" |
|
) |
|
with gr.Row(): |
|
with gr.Column(): |
|
with gr.Row(): |
|
text = gr.Text(label="Input Text", interactive=True) |
|
with gr.Column(): |
|
radio = gr.Radio(["file", "mic"], value="file", |
|
label="Condition on a melody (optional) File or Mic") |
|
melody = gr.Audio(source="upload", type="numpy", label="File", |
|
interactive=True, elem_id="melody-input") |
|
with gr.Row(): |
|
submit = gr.Button("Submit") |
|
|
|
_ = gr.Button("Interrupt").click(fn=interrupt, queue=False) |
|
with gr.Row(): |
|
model = gr.Radio(["melody", "medium", "small", "large"], |
|
label="Model", value="melody", interactive=True) |
|
with gr.Row(): |
|
duration = gr.Slider(minimum=1, maximum=120, value=10, label="Duration", interactive=True) |
|
with gr.Row(): |
|
topk = gr.Number(label="Top-k", value=250, interactive=True) |
|
topp = gr.Number(label="Top-p", value=0, interactive=True) |
|
temperature = gr.Number(label="Temperature", value=1.0, interactive=True) |
|
cfg_coef = gr.Number(label="Classifier Free Guidance", value=3.0, interactive=True) |
|
with gr.Column(): |
|
output = gr.Video(label="Generated Music") |
|
submit.click(predict_full, |
|
inputs=[model, text, melody, duration, topk, topp, temperature, cfg_coef], |
|
outputs=[output]) |
|
radio.change(toggle_audio_src, radio, [melody], queue=False, show_progress=False) |
|
gr.Examples( |
|
fn=predict_full, |
|
examples=[ |
|
[ |
|
"An 80s driving pop song with heavy drums and synth pads in the background", |
|
"./assets/bach.mp3", |
|
"melody" |
|
], |
|
[ |
|
"A cheerful country song with acoustic guitars", |
|
"./assets/bolero_ravel.mp3", |
|
"melody" |
|
], |
|
[ |
|
"90s rock song with electric guitar and heavy drums", |
|
None, |
|
"medium" |
|
], |
|
[ |
|
"a light and cheerly EDM track, with syncopated drums, aery pads, and strong emotions", |
|
"./assets/bach.mp3", |
|
"melody" |
|
], |
|
[ |
|
"lofi slow bpm electro chill with organic samples", |
|
None, |
|
"medium", |
|
], |
|
], |
|
inputs=[text, melody, model], |
|
outputs=[output] |
|
) |
|
gr.Markdown( |
|
""" |
|
### More details |
|
|
|
The model will generate a short music extract based on the description you provided. |
|
The model can generate up to 30 seconds of audio in one pass. It is now possible |
|
to extend the generation by feeding back the end of the previous chunk of audio. |
|
This can take a long time, and the model might lose consistency. The model might also |
|
decide at arbitrary positions that the song ends. |
|
|
|
**WARNING:** Choosing long durations will take a long time to generate (2min might take ~10min). |
|
An overlap of 12 seconds is kept with the previously generated chunk, and 18 "new" seconds |
|
are generated each time. |
|
|
|
We present 4 model variations: |
|
1. Melody -- a music generation model capable of generating music condition |
|
on text and melody inputs. **Note**, you can also use text only. |
|
2. Small -- a 300M transformer decoder conditioned on text only. |
|
3. Medium -- a 1.5B transformer decoder conditioned on text only. |
|
4. Large -- a 3.3B transformer decoder conditioned on text only (might OOM for the longest sequences.) |
|
|
|
When using `melody`, ou can optionaly provide a reference audio from |
|
which a broad melody will be extracted. The model will then try to follow both |
|
the description and melody provided. |
|
|
|
You can also use your own GPU or a Google Colab by following the instructions on our repo. |
|
See [github.com/facebookresearch/audiocraft](https://github.com/facebookresearch/audiocraft) |
|
for more details. |
|
""" |
|
) |
|
|
|
interface.queue().launch(**launch_kwargs) |
|
|
|
|
|
def ui_batched(launch_kwargs): |
|
with gr.Blocks() as demo: |
|
gr.Markdown( |
|
""" |
|
# MusicGen |
|
|
|
This is the demo for [MusicGen](https://github.com/facebookresearch/audiocraft), |
|
a simple and controllable model for music generation |
|
presented at: ["Simple and Controllable Music Generation"](https://huggingface.co/papers/2306.05284). |
|
<br/> |
|
<a href="https://huggingface.co/spaces/facebook/MusicGen?duplicate=true" |
|
style="display: inline-block;margin-top: .5em;margin-right: .25em;" target="_blank"> |
|
<img style="margin-bottom: 0em;display: inline;margin-top: -.25em;" |
|
src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a> |
|
for longer sequences, more control and no queue.</p> |
|
""" |
|
) |
|
with gr.Row(): |
|
with gr.Column(): |
|
with gr.Row(): |
|
text = gr.Text(label="Describe your music", lines=2, interactive=True) |
|
with gr.Column(): |
|
radio = gr.Radio(["file", "mic"], value="file", |
|
label="Condition on a melody (optional) File or Mic") |
|
melody = gr.Audio(source="upload", type="numpy", label="File", |
|
interactive=True, elem_id="melody-input") |
|
with gr.Row(): |
|
submit = gr.Button("Generate") |
|
with gr.Column(): |
|
output = gr.Video(label="Generated Music") |
|
submit.click(predict_batched, inputs=[text, melody], |
|
outputs=[output], batch=True, max_batch_size=MAX_BATCH_SIZE) |
|
radio.change(toggle_audio_src, radio, [melody], queue=False, show_progress=False) |
|
gr.Examples( |
|
fn=predict_batched, |
|
examples=[ |
|
[ |
|
"An 80s driving pop song with heavy drums and synth pads in the background", |
|
"./assets/bach.mp3", |
|
], |
|
[ |
|
"A cheerful country song with acoustic guitars", |
|
"./assets/bolero_ravel.mp3", |
|
], |
|
[ |
|
"90s rock song with electric guitar and heavy drums", |
|
None, |
|
], |
|
[ |
|
"a light and cheerly EDM track, with syncopated drums, aery pads, and strong emotions bpm: 130", |
|
"./assets/bach.mp3", |
|
], |
|
[ |
|
"lofi slow bpm electro chill with organic samples", |
|
None, |
|
], |
|
], |
|
inputs=[text, melody], |
|
outputs=[output] |
|
) |
|
gr.Markdown(""" |
|
### More details |
|
|
|
The model will generate 12 seconds of audio based on the description you provided. |
|
You can optionaly provide a reference audio from which a broad melody will be extracted. |
|
The model will then try to follow both the description and melody provided. |
|
All samples are generated with the `melody` model. |
|
|
|
You can also use your own GPU or a Google Colab by following the instructions on our repo. |
|
|
|
See [github.com/facebookresearch/audiocraft](https://github.com/facebookresearch/audiocraft) |
|
for more details. |
|
""") |
|
|
|
demo.queue(max_size=8 * 4).launch(**launch_kwargs) |
|
|
|
|
|
|
|
def ui_harp(launch_kwargs): |
|
with gr.Blocks() as interface: |
|
gr.Markdown( |
|
""" |
|
# MusicGen |
|
This is your private demo for [MusicGen](https://github.com/facebookresearch/audiocraft), |
|
a simple and controllable model for music generation |
|
presented at: ["Simple and Controllable Music Generation"](https://huggingface.co/papers/2306.05284) |
|
""" |
|
) |
|
with gr.Row(): |
|
with gr.Column(): |
|
with gr.Row(): |
|
melody = gr.Audio(label="Melody Audio", type="filepath") |
|
text = gr.Text(label="Input Text", interactive=True) |
|
|
|
|
|
|
|
|
|
duration = gr.Slider(minimum=1, maximum=120, value=10, label="Duration", interactive=True) |
|
|
|
|
|
|
|
temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=10.0, value=1.0,interactive=True) |
|
cfg_coef = gr.Slider(label="Classifier Free Guidance", minimu=1.0, maximum=5.0, value=3.0, interactive=True) |
|
|
|
inputs = [melody, text, duration, temperature, cfg_coef] |
|
with gr.Column(): |
|
output = gr.Audio(label="Output Audio", type="filepath") |
|
|
|
def predict_harp(_melody, _text, _duration, _temperature, _cfg_coef): |
|
from audiotools import AudioSignal |
|
import torch |
|
|
|
|
|
|
|
output = predict_full( |
|
model="small", |
|
melody=None, |
|
text=_text, duration=_duration, |
|
topk=250, topp=0, |
|
temperature=_temperature, |
|
cfg_coef=_cfg_coef |
|
) |
|
print(output) |
|
sig = AudioSignal(output) |
|
|
|
sig.write('output.wav') |
|
return sig.path_to_file |
|
|
|
|
|
from pyharp import ModelCard, build_endpoint |
|
card = ModelCard( |
|
name="MusicGen (Meta)", |
|
description="The model will generate a short excerpt based on the text description you provided. The model can generate up to 30 seconds of audio in one pass.", |
|
author="Jade Copet et al.", |
|
tags=["example", "music generation"] |
|
) |
|
|
|
build_endpoint(inputs, output, predict_harp, card=card) |
|
|
|
|
|
interface.queue().launch(**launch_kwargs) |
|
|
|
|
|
if __name__ == "__main__": |
|
parser = argparse.ArgumentParser() |
|
parser.add_argument( |
|
'--listen', |
|
type=str, |
|
default='0.0.0.0' if 'SPACE_ID' in os.environ else '127.0.0.1', |
|
help='IP to listen on for connections to Gradio', |
|
) |
|
parser.add_argument( |
|
'--username', type=str, default='', help='Username for authentication' |
|
) |
|
parser.add_argument( |
|
'--password', type=str, default='', help='Password for authentication' |
|
) |
|
parser.add_argument( |
|
'--server_port', |
|
type=int, |
|
default=0, |
|
help='Port to run the server listener on', |
|
) |
|
parser.add_argument( |
|
'--inbrowser', action='store_true', help='Open in browser' |
|
) |
|
parser.add_argument( |
|
'--share', action='store_true', help='Share the gradio UI' |
|
) |
|
|
|
args = parser.parse_args() |
|
|
|
launch_kwargs = {} |
|
launch_kwargs['server_name'] = args.listen |
|
|
|
if args.username and args.password: |
|
launch_kwargs['auth'] = (args.username, args.password) |
|
if args.server_port: |
|
launch_kwargs['server_port'] = args.server_port |
|
if args.inbrowser: |
|
launch_kwargs['inbrowser'] = args.inbrowser |
|
if args.share: |
|
launch_kwargs['share'] = args.share |
|
|
|
ui_harp(launch_kwargs) |
|
|