Spaces:
Sleeping
Sleeping
| import os | |
| import tempfile | |
| import requests | |
| import gradio as gr | |
| import torch | |
| import librosa | |
| import numpy as np | |
| import subprocess | |
| import sys | |
| def install_dependencies(): | |
| """Install required packages for deployment""" | |
| try: | |
| if not os.path.exists('ss-vq-vae'): | |
| print("Cloning ss-vq-vae repository...") | |
| subprocess.run(['git', 'clone', 'https://github.com/cifkao/ss-vq-vae.git'], check=True) | |
| subprocess.run([sys.executable, '-m', 'pip', 'install', './ss-vq-vae/src'], check=True) | |
| print("Dependencies installed successfully!") | |
| except Exception as e: | |
| print(f"Error installing dependencies: {e}") | |
| raise | |
| # Install dependencies for deployment | |
| try: | |
| install_dependencies() | |
| import confugue | |
| from ss_vq_vae.models.vqvae_oneshot import Experiment | |
| except ImportError: | |
| print("ss-vq-vae not found. Please install manually or run in Colab.") | |
| sys.exit(1) | |
| def download_model(): | |
| """Download model files if they don't exist""" | |
| model_dir = 'ss-vq-vae/experiments/model' | |
| os.makedirs(model_dir, exist_ok=True) | |
| model_path = os.path.join(model_dir, 'model_state.pt') | |
| if not os.path.exists(model_path): | |
| print("Downloading model...") | |
| url = 'https://adasp.telecom-paris.fr/rc-ext/demos_companion-pages/vqvae_examples/ssvqvae_model_state.pt' | |
| response = requests.get(url) | |
| with open(model_path, 'wb') as f: | |
| f.write(response.content) | |
| print("Model downloaded successfully!") | |
| # Initialize model | |
| download_model() | |
| logdir = 'ss-vq-vae/experiments/model' | |
| cfg = confugue.Configuration.from_yaml_file(os.path.join(logdir, 'config.yaml')) | |
| exp = cfg.configure(Experiment, logdir=logdir, device='cpu') | |
| exp.model.load_state_dict(torch.load(os.path.join(logdir, 'model_state.pt'), map_location=exp.device)) | |
| exp.model.train(False) | |
| # Preset audio URLs | |
| INPUT_ROOT = 'https://adasp.telecom-paris.fr/rc-ext/demos_companion-pages/vqvae_examples/' | |
| INPUT_URLS = { | |
| 'Electric Guitar': INPUT_ROOT + 'real/content/UnicornRodeo_Maybe_UnicornRodeo_Maybe_Full_25_ElecGtr2CloseMic3.0148.mp3', | |
| 'Electric Organ': INPUT_ROOT + 'real/style/AllenStone_Naturally_Allen%20Stone_Naturally_Keys-Organ-Active%20DI.0253.mp3', | |
| 'Jazz Piano': INPUT_ROOT + 'real/style/MaurizioPagnuttiSextet_AllTheGinIsGone_MaurizioPagnuttiSextet_AllTheGinIsGone_Full_12_PianoMics1.08.mp3', | |
| 'Synth': INPUT_ROOT + 'real/content/Skelpolu_TogetherAlone_Skelpolu_TogetherAlone_Full_13_Synth.0190.mp3', | |
| 'Rhodes DI': INPUT_ROOT + 'real/content/Diesel13_ColourMeRed_Diesel13_ColourMeRed_Full_30_RhodesDI.0062.mp3', | |
| 'Acoustic Guitar Lead': INPUT_ROOT + 'real/style/NikolaStajicFtVlasisKostas_Nalim_Nikola%20Stajic%20ft.%20Vlasis%20Kostas_Nalim_Acoustic%20Guitar-Lead-Ela%20M%20251.0170.mp3', | |
| 'Bass Amp': INPUT_ROOT + 'real/content/HurrayForTheRiffRaff_LivingInTheCity_Hurray%20for%20the%20Riff%20Raff_Livin%20in%20the%20City_Bass-Amp-M82.0018.mp3', | |
| 'Bass Bip': INPUT_ROOT + 'real/style/RememberDecember_CUNextTime_RememberDecember_CUNextTime_Full_11_Bass_bip.041.mp3', | |
| 'SynthFX': INPUT_ROOT + 'real/content/MR0902_JamesElder_MR0902_JamesElder_Full_13_SynthFX1.163.mp3', | |
| 'Electric Guitar Close': INPUT_ROOT + 'real/style/Fergessen_TheWind_Fergessen_TheWind_Full_17_SlecGtr3a_Close.146.mp3', | |
| 'Rhodes NBATG': INPUT_ROOT + 'real/content/NickiBluhmAndTheGramblers_GoGoGo_NBATG%20-%20Rhodes%20-%20DI.098.mp3', | |
| 'Keys DI Grace': INPUT_ROOT + 'real/style/JessicaChildress_SlowDown_SD%20KEYS-DI-GRACE.147.mp3', | |
| 'Dulcimer': INPUT_ROOT + 'real/content/ButterflyEffect_PreachRightHere_ButterflyEffect_PreachRightHere_Full_16_Dulcimer2.076.mp3', | |
| 'Strings Section': INPUT_ROOT + 'real/style/AngeloBoltini_ThisTown_AngeloBoltini_ThisTown_Full_47_Strings_SectionMic_Vln2.0181.mp3', | |
| 'Mellotron': INPUT_ROOT + 'real/content/Triviul_Dorothy_Triviul_Dorothy_Full_07_Mellotron.120.mp3', | |
| 'Acoustic Guitar CU': INPUT_ROOT + 'real/style/UncleDad_WhoIAm_legend-strings_AC%20GUITAR-3-CU29-SHADOWHILL.R.0106.mp3', | |
| 'Fiddle': INPUT_ROOT + 'real/content/EndaReilly_CurAnLongAgSeol_EndaReilly_CurAnLongAgSeol_Full_10_Fiddle2.0163.mp3', | |
| 'Violins': INPUT_ROOT + 'real/style/ScottElliott_AeternumVale_ScottElliott_AeternumVale_Full_41_Violins.0138.mp3', | |
| 'Upright Bass': INPUT_ROOT + 'real/content/AbletonesBigBand_SongOfIndia_UPRIGHT%20BASS%20-%20ELA%20M%20260%20-%20Neve%2033102.136.mp3', | |
| 'Taiko': INPUT_ROOT + 'real/style/CarlosGonzalez_APlaceForUs_CarlosGonzalez_APlaceForUs_Full_21_Taiko.0115.mp3', | |
| 'Guitar 2': INPUT_ROOT + 'real/content/AllHandsLost_Ambitions_AllHandsLost_Ambitions_Full_Guitar%202.0292.mp3', | |
| 'Alto Sax': INPUT_ROOT + 'real/style/SunshineGarciaBand_ForIAmTheMoon_zip5-outro-uke-shaker_OUTRO%20ALTO-251E-SSL6000E.0290.mp3', | |
| 'Bass Close Mic': INPUT_ROOT + 'real/content/DonCamilloChoir_MarshMarigoldsSong_DonCamilloChoir_MarshMarigoldsSong_Full_08_BassCloseMic2.000.mp3', | |
| 'Electric Guitar Distorted': INPUT_ROOT + 'real/style/EnterTheHaggis_TwoBareHands_25.%20Jubilee%20Riots%20-%202%20Bar%20Hands_ELE%20Guitars-Ignater-M81.160.mp3', | |
| 'Bells': INPUT_ROOT + 'real/content/cryonicPAX_Melancholy_cryonicPAX_Melancholy_Full_10_Bells.0034.mp3', | |
| 'Bass Mic 647': INPUT_ROOT + 'real/style/KungFu_JoyRide_40.%20Kung%20Fu%20-%20Joy%20ride_Bass-Mic-647.0090.mp3', | |
| } | |
| # Separate content and style options based on URL paths | |
| CONTENT_OPTIONS = [key for key in INPUT_URLS.keys() if any(word in INPUT_URLS[key] for word in ['content'])] | |
| STYLE_OPTIONS = [key for key in INPUT_URLS.keys() if any(word in INPUT_URLS[key] for word in ['style'])] | |
| # Add remaining items to both lists if they don't contain 'content' or 'style' | |
| for key in INPUT_URLS.keys(): | |
| if key not in CONTENT_OPTIONS and key not in STYLE_OPTIONS: | |
| CONTENT_OPTIONS.append(key) | |
| STYLE_OPTIONS.append(key) | |
| def load_audio_from_url(url, sr=None): | |
| """Load audio from URL by downloading to temporary file""" | |
| response = requests.get(url) | |
| with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as tmp_file: | |
| tmp_file.write(response.content) | |
| tmp_file_path = tmp_file.name | |
| audio, _ = librosa.load(tmp_file_path, sr=sr) | |
| os.unlink(tmp_file_path) | |
| return audio | |
| def preview_content_preset(preset_name): | |
| """Load and return audio for content preset preview""" | |
| if preset_name and preset_name in INPUT_URLS: | |
| try: | |
| audio = load_audio_from_url(INPUT_URLS[preset_name], sr=exp.sr) | |
| preview_duration = 5 | |
| max_samples = int(preview_duration * exp.sr) | |
| if len(audio) > max_samples: | |
| audio = audio[:max_samples] | |
| return (exp.sr, audio) | |
| except Exception as e: | |
| print(f"Error loading content preset: {e}") | |
| return None | |
| return None | |
| def preview_style_preset(preset_name): | |
| """Load and return audio for style preset preview""" | |
| if preset_name and preset_name in INPUT_URLS: | |
| try: | |
| audio = load_audio_from_url(INPUT_URLS[preset_name], sr=exp.sr) | |
| preview_duration = 5 | |
| max_samples = int(preview_duration * exp.sr) | |
| if len(audio) > max_samples: | |
| audio = audio[:max_samples] | |
| return (exp.sr, audio) | |
| except Exception as e: | |
| print(f"Error loading style preset: {e}") | |
| return None | |
| return None | |
| def process_timbre_transfer(content_file, content_preset, style_file, style_preset, max_duration=8): | |
| """Process timbre transfer with uploaded files or presets""" | |
| try: | |
| # Load content audio | |
| if content_file is not None: | |
| a_content, _ = librosa.load(content_file, sr=exp.sr) | |
| elif content_preset and content_preset in INPUT_URLS: | |
| a_content = load_audio_from_url(INPUT_URLS[content_preset], sr=exp.sr) | |
| else: | |
| return None, "Please upload a content file or select a content preset" | |
| # Load style audio | |
| if style_file is not None: | |
| a_style, _ = librosa.load(style_file, sr=exp.sr) | |
| elif style_preset and style_preset in INPUT_URLS: | |
| a_style = load_audio_from_url(INPUT_URLS[style_preset], sr=exp.sr) | |
| else: | |
| return None, "Please upload a style file or select a style preset" | |
| # Limit duration | |
| max_samples = int(max_duration * exp.sr) | |
| if len(a_content) > max_samples: | |
| a_content = a_content[:max_samples] | |
| if len(a_style) > max_samples: | |
| a_style = a_style[:max_samples] | |
| # Preprocess | |
| s_content = torch.as_tensor(exp.preprocess(a_content), device=exp.device)[None, :] | |
| s_style = torch.as_tensor(exp.preprocess(a_style), device=exp.device)[None, :] | |
| l_content, l_style = (torch.as_tensor([x.shape[2]], device=exp.device) for x in [s_content, s_style]) | |
| # Run model | |
| with torch.no_grad(): | |
| s_output = exp.model(input_c=s_content, input_s=s_style, | |
| length_c=l_content, length_s=l_style) | |
| # Postprocess | |
| a_output = exp.postprocess(s_output.cpu().numpy()[0]) | |
| return (exp.sr, a_output), "Transfer completed successfully!" | |
| except Exception as e: | |
| return None, f"Error: {str(e)}" | |
| # Create Gradio interface | |
| with gr.Blocks(title="VQ-VAE Timbre Transfer", theme=gr.themes.Soft()) as demo: | |
| gr.Markdown(""" | |
| # 🎵 VQ-VAE Timbre Transfer Demo | |
| Transfer the timbre (tone/texture) from one audio source to another while preserving the musical content. | |
| **Content**: Musical notes/melody that will be preserved | |
| **Style**: Instrument timbre/texture that will be applied | |
| ### How It Works: | |
| The model separates musical content from timbre, then recombines them. Some instrument combinations work better than others due to harmonic compatibility and spectral characteristics. | |
| ### Tips: | |
| - Listen to preset previews to understand the source material | |
| - Harmonic instruments (piano, guitar) often transfer well to each other | |
| - Try different combinations - unexpected results can be musically interesting! | |
| """) | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("### 🎼 Content Audio") | |
| content_file = gr.Audio(label="Upload Content Audio", type="filepath") | |
| content_preset = gr.Dropdown( | |
| choices=[""] + CONTENT_OPTIONS, | |
| label="Or choose preset", | |
| value="" | |
| ) | |
| content_preview = gr.Audio( | |
| label="🔊 Content Preview (5s)", | |
| interactive=False, | |
| visible=False | |
| ) | |
| with gr.Column(): | |
| gr.Markdown("### 🎨 Style Audio") | |
| style_file = gr.Audio(label="Upload Style Audio", type="filepath") | |
| style_preset = gr.Dropdown( | |
| choices=[""] + STYLE_OPTIONS, | |
| label="Or choose preset", | |
| value="Electric Guitar Close" | |
| ) | |
| style_preview = gr.Audio( | |
| label="🔊 Style Preview (5s)", | |
| interactive=False, | |
| visible=True | |
| ) | |
| max_duration = gr.Slider(1, 15, value=8, step=1, label="Max Duration (seconds)") | |
| process_btn = gr.Button("🚀 Transfer Timbre", variant="primary", size="lg") | |
| with gr.Row(): | |
| output_audio = gr.Audio(label="🎵 Output Audio", interactive=False) | |
| status_msg = gr.Textbox(label="Status", interactive=False, max_lines=3) | |
| # Hide previews when user uploads files | |
| content_file.change( | |
| fn=lambda file: gr.update(visible=False) if file is not None else None, | |
| inputs=[content_file], | |
| outputs=[content_preview] | |
| ) | |
| style_file.change( | |
| fn=lambda file: gr.update(visible=False) if file is not None else None, | |
| inputs=[style_file], | |
| outputs=[style_preview] | |
| ) | |
| # Show previews when presets are selected | |
| content_preset.change( | |
| fn=lambda preset, file: ( | |
| preview_content_preset(preset) if preset and file is None else None, | |
| gr.update(visible=bool(preset and file is None)) | |
| ), | |
| inputs=[content_preset, content_file], | |
| outputs=[content_preview, content_preview] | |
| ) | |
| style_preset.change( | |
| fn=lambda preset, file: ( | |
| preview_style_preset(preset) if preset and file is None else None, | |
| gr.update(visible=bool(preset and file is None)) | |
| ), | |
| inputs=[style_preset, style_file], | |
| outputs=[style_preview, style_preview] | |
| ) | |
| # Load default style preview | |
| demo.load( | |
| fn=lambda: preview_style_preset("Electric Guitar Close"), | |
| outputs=[style_preview] | |
| ) | |
| # Process button | |
| process_btn.click( | |
| fn=process_timbre_transfer, | |
| inputs=[content_file, content_preset, style_file, style_preset, max_duration], | |
| outputs=[output_audio, status_msg] | |
| ) | |
| gr.Markdown(""" | |
| ### 🔧 Troubleshooting | |
| - **Poor transfer quality?** Try different instrument combinations or adjust max duration | |
| - **Audio doesn't load?** Check internet connection or try different presets | |
| - **Processing slow?** Reduce max duration or try shorter audio clips | |
| ### 📖 Citation | |
| Original work by Ondřej Cífka (InterDigital R&D and Télécom Paris, 2020). | |
| Demo by Ali Dulaimi. | |
| """) | |
| if __name__ == "__main__": | |
| demo.launch(share=True, debug=True, height=1400) |