import time import os import pickle import sys sys.path.append('/') from src.music.pipeline.music_pipeline import encode_music from src.music2cocktailrep.pipeline.music2cocktailrep import music2cocktailrep, setup_translation_models, debug_translation from src.cocktails.pipeline.cocktailrep2recipe import cocktailrep2recipe from src.debugger import Debugger from datetime import datetime from shutil import copy import streamlit as st from src.music.config import AUDIO_PATH, MIDI_PATH from pretty_midi import PrettyMIDI import numpy as np import pydub from PIL import Image st.set_page_config( page_title="Test", page_icon="🎹", ) st.title('TastyPiano') synestesia_path = 'data/synesthesia/' debugger = Debugger(verbose=False) def centered_module(func, text, args=None): _, col2, _ = st.columns([1, 2, 1]) with col2: return func(text) def centered_button(func, text, args=None): _, _, _, col3, _, _, _ = st.columns([1, 1, 1, 1, 1, 1, 1]) with col3: return func(text) def setup_streamlite(): setup_translation_models() image = Image.open('./data/pianocktail.jpg') st.image(image, caption='Pianocktail by Compagnie la Rumeur') st.subheader("Ready to taste music?") st.markdown("TastyPiano generates music--taste synesthetic experiences by letting you turn any piano song into a cocktail. It is inspired by the" " *Pianocktail*, a literary invention from the French novelist, jazz musician, singer and engineer Boris Vian. I see TastyPiano as a digital member of the " "Pianocktail species, along with [other](https://www.youtube.com/watch?v=5bdX5i0nAWw) " "[wonderful](https://www.youtube.com/watch?v=pzsDOH-xtrs&list=PLC196AA37A2D1C066&index=3) [machines](https://www.youtube.com/watch?v=y0RJg7I2x34).") st.markdown("But TastyPiano is different from existing pianocktails. While existing version merely map played notes to drops of corresponding ingredients, " "TastyPiano listens to the song, analyzes the music, hallucinates matching tastes and flavors, before it starts generating cocktail recipes " "until it finds the one matching its gustatory hallucination. Check my [blog post](https://ccolas.github.io/project/pianocktail) for more information." " Hear and taste it for yourself!") st.subheader("How to use it?") st.markdown("Provide a piano solo input and click on the **Taste it!** button.\n" "\nYou can input either: \n* a YouTube url **or**\n* an audio file (.mp3) **or**\n* a midi file (.mid)\n" "All these should be **only piano**, no other instrument.\n" "Note that audio sources are cropped to the first 40s because the process of " "converting it to midi is rather slow (1sec/sec). Midi inputs are taken whole. Please report any bug / buggy url in the community tab.") st.subheader("Prepare") col1, col2, col3 = st.columns(3) generate_audio_from_midi = False with col1: st.markdown('**YouTube url**') url = st.text_area('Type it below', 'https://www.youtube.com/watch?v=UGK70IkP830', height=160) with col2: st.markdown('**Audio file**') audio = st.file_uploader("Upload it here (.mp3)", type=['.mp3']) with col3: st.markdown('**Midi file**') midi = st.file_uploader("Upload it here (.mid)", type=['.mid']) generate_audio_from_midi = st.checkbox('Generate audio? Untick if the song is too long (>10min)', value=True) def run(unit): setup_and_run(unit=unit, url=url, midi=midi, audio=audio, generate_audio_from_midi=generate_audio_from_midi, extra_code=None) # url = "https://youtu.be/VUCI-1vIbUo" # unit = 'mL' # run(unit) st.markdown('##') unit = st.radio('Pick the units (before pressing "Taste it!", default mL)', ['mL', 'oz'], index=0) button = centered_button(st.button, 'Taste it!') if button: run(unit) def pianocktail(unit='mL', record=False, url=None, midi=None, audio=None, processed=None, crop=40, verbose=False, debug=False, level=0): assert url is not None or midi is not None or audio is not None or processed is not None if verbose: print('------\nNew synesthetic exploration!') init_time = time.time() try: with st.spinner("Listening to the song (~20s).."): music_ai_rep, music_handcoded_rep, all_paths, error = encode_music(record=record, url=url, audio_path=audio, midi_path=midi, nb_aug=0, noise_injection=False, augmentation=False, processed_path=processed, crop=crop, apply_filtering=False, verbose=verbose, level=level+2) if music_ai_rep is not None: with st.spinner(text="Thinking about corresponding flavors.."): cocktail_rep, affective_cluster_id, affect = music2cocktailrep(music_ai_rep, music_handcoded_rep, verbose=verbose, level=level+2) with st.spinner("Trying recipes (15s).."): cocktail_recipes, scores = cocktailrep2recipe(cocktail_rep, unit=unit, target_affective_cluster=affective_cluster_id, verbose=verbose, full_verbose=verbose, \ level=level+2) cocktail_recipe = cocktail_recipes[0] recipe_score = scores[0] if debug: music_reconstruction = debug_translation(music_ai_rep) debugger.extract_info(all_paths, affective_cluster_id, affect, cocktail_rep, music_reconstruction, recipe_score, verbose=verbose, level=level+2) debug_info = debugger.debug_dict else: debug_info = None if verbose: print(cocktail_recipe.replace('Recipe', ' ' * (level + 2) + 'Generated recipe:').replace('None ()', '')) debugger.print_debug(level=level+2) print(f'\nEnd of synesthetic exploration ({int(time.time() - init_time)} secs).\n------') st.success('Recipe found!') else: st.error('Error in listening. Is the url valid? the audio an .mp3? the midi a .mid?') cocktail_recipe = None debug_info = None except: st.error('Error: ' + error) cocktail_recipe = None debug_info = None return cocktail_recipe, debug_info def setup_and_run(unit='mL', url=None, midi=None, audio=None, generate_audio_from_midi=False, verbose=True, debug=True, extra_code=None): if url is None and midi is None and audio is None: st.error('Please enter a piano input.') assert False st.subheader('Synesthesia') now = datetime.now() folder_name = f'{now.year}-{now.month}-{now.day}_{now.hour}:{now.minute}:{now.second}' folder_path = synestesia_path + folder_name if extra_code is not None: folder_path += '_' + extra_code if os.path.exists(folder_path): folder_path += '_2' folder_path += '/' os.makedirs(folder_path, exist_ok=True) if midi is not None: st.write(f' \tReading from midi file: {midi.name}') midi_path = MIDI_PATH + 'from_url_midi/' + midi.name[:-4] + '_midi.mid' audio_path = AUDIO_PATH + 'from_url/' + midi.name.replace('.mid', '.mp3') with open(midi_path, "wb") as f: f.write(midi.getbuffer()) midi = midi_path if generate_audio_from_midi: midi_data = PrettyMIDI(midi_path) audio_data = midi_data.fluidsynth(fs=44100) y = np.int16(audio_data * 2 ** 15) song = pydub.AudioSegment(y.tobytes(), frame_rate=44100, sample_width=2, channels=1) song.export(audio_path, format="mp3", bitrate="320k") # st.write(audio_data) st.audio(audio_path, format='audio/mp3') url = None elif audio is not None: st.write(f' \tReading from audio file: {audio.name}') audio_path = AUDIO_PATH + 'from_url/' + audio.name with open(audio_path, "wb") as f: f.write(audio.getbuffer()) audio = audio_path audio_file = open(audio, 'rb') audio_bytes = audio_file.read() st.audio(audio_bytes, format='audio/mp3') url = None else: st.write(f' \tReading from YouTube url: {url}') st.video(url) _, col2, _ = st.columns([1, 1, 1]) with col2: st.markdown('##') recipe, debug = pianocktail(unit=unit, url=url, midi=midi, audio=audio, verbose=verbose, debug=debug) with open(folder_path + 'debug.pk', 'wb') as f: pickle.dump(debug, f) with open(folder_path + 'recipe.txt', 'w') as f: f.write(recipe) paths = debug['all_paths'] if paths['url'] is not None: with open(folder_path + 'url.txt', 'w') as f: f.write(paths['url']) for k in ['audio_path', 'midi_path']: origin = paths[k] if origin is not None: copy(origin, folder_path + origin.split('/')[-1]) st.subheader('Recipe') recipe = recipe.replace(' Enjoy!', ' \nEnjoy!').replace('\n', ' \n') st.text(recipe) st.markdown('**About this synesthesia**') closest_songs = [debug['nn_music'][i][:-26].split('structured_')[1].replace('_', ' ') for i in range(3)] str_songs = 'These are the closest song I know: ' str_songs += ' '.join([f' \n* {closest_songs[i]}' for i in range(3)]) st.markdown(str_songs + '.') str_cocktails = 'These are existing cocktails that are close to the taste of this song:' str_cocktails += ' '.join([f' \n* {cocktail_name}: {cocktail_url}' for cocktail_name, cocktail_url in zip(debug['nearest_cocktail_names'][:3], debug['nearest_cocktail_urls'][:3])]) st.markdown(str_cocktails + '.') if __name__ == '__main__': setup_streamlite() # urls = ["https://www.youtube.com/watch?v=PLFVGwGQcB0", # "https://www.youtube.com/watch?v=VQmuAr93OlI", # "https://www.youtube.com/watch?v=Nv2GgV34qIg&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=4", # "https://www.youtube.com/watch?v=qAEIjWYdoYc&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=1", # "https://www.youtube.com/watch?v=M73x3O7dhmg&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=5"] # setup_translation_models() # setup_and_run(url=urls[0], verbose=True, debug=True) # recipes = [] # for url in urls: # recipe = pianocktail(url=url, verbose=True, debug=True)[0] # recipes.append(recipe) # stop = 1