Process longer Audio

.gitignore

@@ -53,3 +53,4 @@ ENV/
 /notebooks
 /local_scripts
 /notes
+/.vs

app.py

@@ -13,6 +13,8 @@ import gradio as gr
 import os
 from audiocraft.models import MusicGen
 from audiocraft.data.audio import audio_write
+from audiocraft.utils.extend import generate_music_segments
+import numpy as np
 
 MODEL = None
 IS_SHARED_SPACE = "musicgen/MusicGen" in os.environ.get('SPACE_ID', '')
@@ -30,32 +32,47 @@ def predict(model, text, melody, duration, topk, topp, temperature, cfg_coef):
         MODEL = load_model(model)
 
     if duration > MODEL.lm.cfg.dataset.segment_duration:
-        raise gr.Error("MusicGen currently supports durations of up to 30 seconds!")
+        segment_duration = MODEL.lm.cfg.dataset.segment_duration
+    else:
+        segment_duration = duration
     MODEL.set_generation_params(
         use_sampling=True,
         top_k=topk,
         top_p=topp,
         temperature=temperature,
         cfg_coef=cfg_coef,
-        duration=duration,
+        duration=segment_duration,
     )
 
     if melody:
-        sr, melody = melody[0], torch.from_numpy(melody[1]).to(MODEL.device).float().t().unsqueeze(0)
-        print(melody.shape)
-        if melody.dim() == 2:
-            melody = melody[None]
-        melody = melody[..., :int(sr * MODEL.lm.cfg.dataset.segment_duration)]
-        output = MODEL.generate_with_chroma(
-            descriptions=[text],
-            melody_wavs=melody,
-            melody_sample_rate=sr,
-            progress=False
-        )
+        if duration > MODEL.lm.cfg.dataset.segment_duration:
+            output_segments = generate_music_segments(text, melody, MODEL, duration, MODEL.lm.cfg.dataset.segment_duration)
+        else:
+            # pure original code
+            sr, melody = melody[0], torch.from_numpy(melody[1]).to(MODEL.device).float().t().unsqueeze(0)
+            print(melody.shape)
+            if melody.dim() == 2:
+                melody = melody[None]
+            melody = melody[..., :int(sr * MODEL.lm.cfg.dataset.segment_duration)]
+            output = MODEL.generate_with_chroma(
+                descriptions=[text],
+                melody_wavs=melody,
+                melody_sample_rate=sr,
+                progress=True
+            )
     else:
         output = MODEL.generate(descriptions=[text], progress=False)
 
-    output = output.detach().cpu().float()[0]
+    if output_segments:
+        try:
+            # Combine the output segments into one long audio file
+            output_segments = [segment.detach().cpu().float()[0] for segment in output_segments]
+            output = torch.cat(output_segments, dim=2)
+        except Exception as e:
+            print(f"error combining segments: {e}. Using first segment only")
+            output = output_segments[0].detach().cpu().float()[0]
+    else:
+        output = output.detach().cpu().float()[0]
     with NamedTemporaryFile("wb", suffix=".wav", delete=False) as file:
         audio_write(
             file.name, output, MODEL.sample_rate, strategy="loudness",
@@ -91,7 +108,7 @@ def ui(**kwargs):
                 with gr.Row():
                     model = gr.Radio(["melody", "medium", "small", "large"], label="Model", value="melody", interactive=True)
                 with gr.Row():
-                    duration = gr.Slider(minimum=1, maximum=30, value=10, label="Duration", interactive=True)
+                    duration = gr.Slider(minimum=1, maximum=1000, value=10, label="Duration", interactive=True)
                 with gr.Row():
                     topk = gr.Number(label="Top-k", value=250, interactive=True)
                     topp = gr.Number(label="Top-p", value=0, interactive=True)
@@ -194,7 +211,7 @@ if __name__ == "__main__":
     parser.add_argument(
         '--server_port',
         type=int,
-        default=0,
+        default=7859,
         help='Port to run the server listener on',
     )
     parser.add_argument(

audiocraft/utils/extend.py

@@ -0,0 +1,111 @@
+import torch
+import math
+from audiocraft.models import MusicGen
+import numpy as np
+
+
+def separate_audio_segments(audio, segment_duration=30):
+    sr, audio_data = audio[0], audio[1]
+    
+    total_samples = len(audio_data)
+    segment_samples = sr * segment_duration
+    
+    total_segments = math.ceil(total_samples / segment_samples)
+    
+    segments = []
+    
+    for segment_idx in range(total_segments):
+        print(f"Audio Input segment {segment_idx + 1} / {total_segments + 1} \r")
+        start_sample = segment_idx * segment_samples
+        end_sample = (segment_idx + 1) * segment_samples
+        
+        segment = audio_data[start_sample:end_sample]
+        segments.append((sr, segment))
+    
+    return segments
+
+def generate_music_segments(text, melody, MODEL, duration:int=10, segment_duration:int=30):
+    # generate audio segments
+    melody_segments = separate_audio_segments(melody, segment_duration) 
+    
+    # Create a list to store the melody tensors for each segment
+    melodys = []
+    output_segments = []
+    
+    # Calculate the total number of segments
+    total_segments = max(math.ceil(duration / segment_duration),1)
+    print(f"total Segments to Generate: {total_segments} for {duration} seconds. Each segment is {segment_duration} seconds")
+
+    # If melody_segments is shorter than total_segments, repeat the segments until the total number of segments is reached
+    if len(melody_segments) < total_segments:
+        for i in range(total_segments - len(melody_segments)):
+            segment = melody_segments[i]
+            melody_segments.append(segment)
+        print(f"melody_segments: {len(melody_segments)} fixed")
+
+    # Iterate over the segments to create list of Meldoy tensors
+    for segment_idx in range(total_segments):
+        print(f"segment {segment_idx} of {total_segments} \r")
+        sr, verse = melody_segments[segment_idx][0], torch.from_numpy(melody_segments[segment_idx][1]).to(MODEL.device).float().t().unsqueeze(0)
+
+        print(f"shape:{verse.shape} dim:{verse.dim()}")
+        if verse.dim() == 2:
+            verse = verse[None]
+        verse = verse[..., :int(sr * MODEL.lm.cfg.dataset.segment_duration)]
+        # Append the segment to the melodys list
+        melodys.append(verse)
+
+    for idx, verse in enumerate(melodys):
+        print(f"Generating New Melody Segment {idx + 1}: {text}\r")
+        output = MODEL.generate_with_chroma(
+            descriptions=[text],
+            melody_wavs=verse,
+            melody_sample_rate=sr,
+            progress=True
+        )
+
+        # Append the generated output to the list of segments
+        #output_segments.append(output[:, :segment_duration])
+        output_segments.append(output)
+        print(f"output_segments: {len(output_segments)}: shape[0]: {output.shape} dim {output.dim()}")
+    return output_segments
+
+#def generate_music_segments(text, melody, duration, MODEL, segment_duration=30):
+#    sr, melody = melody[0], torch.from_numpy(melody[1]).to(MODEL.device).float().t().unsqueeze(0)
+    
+#    # Create a list to store the melody tensors for each segment
+#    melodys = []
+    
+#    # Calculate the total number of segments
+#    total_segments = math.ceil(melody.shape[1] / (sr * segment_duration))
+
+#    # Iterate over the segments
+#    for segment_idx in range(total_segments):
+#        print(f"segment {segment_idx + 1} / {total_segments + 1} \r")
+#        start_frame = segment_idx * sr * segment_duration
+#        end_frame = (segment_idx + 1) * sr * segment_duration
+
+#        # Extract the segment from the melody tensor
+#        segment = melody[:, start_frame:end_frame]
+
+#        # Append the segment to the melodys list
+#        melodys.append(segment)
+
+#    output_segments = []
+
+#    for segment in melodys:
+#        output = MODEL.generate_with_chroma(
+#            descriptions=[text],
+#            melody_wavs=segment,
+#            melody_sample_rate=sr,
+#            progress=False
+#        )
+
+#        # Append the generated output to the list of segments
+#        output_segments.append(output[:, :segment_duration])
+
+#    return output_segments
+
+
+
+

web-ui.bat

@@ -0,0 +1 @@
+py -m app