{ "cells": [ { "cell_type": "code", "execution_count": 108, "id": "61e10139", "metadata": {}, "outputs": [], "source": [ "import pickle\n", "import pretty_midi\n", "from music21 import *\n", "from midi2audio import FluidSynth" ] }, { "cell_type": "code", "execution_count": 48, "id": "1a2b28be", "metadata": {}, "outputs": [], "source": [ "import torch\n", "import torch.nn as nn\n", "from torch.nn import functional as F\n", "\n", "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", "\n", "class GenerationRNN(nn.Module):\n", " def __init__(self, input_size, hidden_size, output_size, n_layers=1):\n", " super(GenerationRNN, self).__init__()\n", " self.input_size = input_size\n", " self.hidden_size = hidden_size\n", " self.output_size = output_size\n", " self.n_layers = n_layers\n", " \n", " self.embedding = nn.Embedding(input_size, hidden_size)\n", " self.gru = nn.GRU(hidden_size, hidden_size, n_layers)\n", " self.decoder = nn.Linear(hidden_size * n_layers, output_size)\n", " \n", " def forward(self, input, hidden):\n", " # Creates embedding of the input texts\n", " #print('initial input', input.size())\n", " input = self.embedding(input.view(1, -1))\n", " #print('input after embedding', input.size())\n", " output, hidden = self.gru(input, hidden)\n", " #print('output after gru', output.size())\n", " #print('hidden after gru', hidden.size())\n", " output = self.decoder(hidden.view(1, -1))\n", " #print('output after decoder', output.size())\n", " return output, hidden\n", "\n", " def init_hidden(self):\n", " return torch.zeros(self.n_layers, 1, self.hidden_size).to(device)" ] }, { "cell_type": "code", "execution_count": 49, "id": "5b7120cf", "metadata": {}, "outputs": [], "source": [ "def predict_multimomial(net, prime_seq, predict_len, temperature=0.8):\n", " '''\n", " Arguments:\n", " prime_seq - priming sequence (converted t)\n", " predict_len - number of notes to predict for after prime sequence\n", " '''\n", " hidden = net.init_hidden()\n", "\n", " predicted = prime_seq.copy()\n", " prime_seq = torch.tensor(prime_seq, dtype = torch.long).to(device)\n", "\n", "\n", " # \"Building up\" the hidden state using the prime sequence\n", " for p in range(len(prime_seq) - 1):\n", " input = prime_seq[p]\n", " _, hidden = net(input, hidden)\n", " \n", " # Last character of prime sequence\n", " input = prime_seq[-1]\n", " \n", " # For every index to predict\n", " for p in range(predict_len):\n", "\n", " # Pass the inputs to the model - output has dimension n_pitches - scores for each of the possible characters\n", " output, hidden = net(input, hidden)\n", " # Sample from the network output as a multinomial distribution\n", " output = output.data.view(-1).div(temperature).exp()\n", " predicted_id = torch.multinomial(output, 1)\n", "\n", " # Add predicted index to the list and use as next input\n", " predicted.append(predicted_id.item()) \n", " input = predicted_id\n", "\n", " return predicted" ] }, { "cell_type": "code", "execution_count": 63, "id": "8ce30142", "metadata": {}, "outputs": [], "source": [ "file_path = './objects/model_cpu.pkl'\n", "with open(file_path, 'rb') as f:\n", " model = pickle.load(f)" ] }, { "cell_type": "code", "execution_count": 64, "id": "84a2ea9b", "metadata": {}, "outputs": [], "source": [ "file_path = './objects/int_to_note.pkl'\n", "with open(file_path, 'rb') as f:\n", " int_to_note = pickle.load(f)" ] }, { "cell_type": "code", "execution_count": 65, "id": "102cd217", "metadata": {}, "outputs": [], "source": [ "file_path = './objects/note_to_int.pkl'\n", "with open(file_path, 'rb') as f:\n", " note_to_int = pickle.load(f)" ] }, { "cell_type": "code", "execution_count": 66, "id": "07815507", "metadata": {}, "outputs": [], "source": [ "def create_midi(prediction_output):\n", " \"\"\" convert the output from the prediction to notes and create a midi file\n", " from the notes \"\"\"\n", " offset = 0\n", " output_notes = []\n", "\n", " # create note and chord objects based on the values generated by the model\n", " for pattern in prediction_output:\n", " # pattern is a chord\n", " if ('.' in pattern) or pattern.isdigit():\n", " notes_in_chord = pattern.split('.')\n", " notes = []\n", " for current_note in notes_in_chord:\n", " new_note = note.Note(int(current_note))\n", " new_note.storedInstrument = instrument.Piano()\n", " notes.append(new_note)\n", " new_chord = chord.Chord(notes)\n", " new_chord.offset = offset\n", " output_notes.append(new_chord)\n", " # pattern is a note\n", " else:\n", " new_note = note.Note(pattern)\n", " new_note.offset = offset\n", " new_note.storedInstrument = instrument.Piano()\n", " output_notes.append(new_note)\n", "\n", " # increase offset each iteration so that notes do not stack\n", " offset += 0.5\n", "\n", " midi_stream = stream.Stream(output_notes)\n", "\n", " return midi_stream" ] }, { "cell_type": "code", "execution_count": 113, "id": "ad197703", "metadata": {}, "outputs": [], "source": [ "def get_note_names(midi):\n", " s2 = instrument.partitionByInstrument(midi)\n", "\n", " piano_part = None\n", " # Filter for only the piano part\n", " instr = instrument.Piano\n", " for part in s2:\n", " if isinstance(part.getInstrument(), instr):\n", " piano_part = part\n", "\n", " notes_song = []\n", " if not piano_part: # Some songs somehow have no piano parts\n", " # Just take the first part\n", " piano_part = s2[0]\n", " \n", " for element in piano_part:\n", " if isinstance(element, note.Note):\n", " # Return the pitch of the single note\n", " notes_song.append(str(element.pitch))\n", " elif isinstance(element, chord.Chord):\n", " # Returns the normal order of a Chord represented in a list of integers\n", " notes_song.append('.'.join(str(n) for n in element.normalOrder))\n", " \n", " return notes_song" ] }, { "cell_type": "code", "execution_count": 114, "id": "0d1140d7", "metadata": {}, "outputs": [], "source": [ "def process_input(input_midi_file, input_randomness, input_duration):\n", " midi = converter.parse(input_midi_file)\n", " note_names = get_note_names(midi)\n", " int_notes = [note_to_int[note_name] for note_name in note_names]\n", " \n", " generated_seq_multinomial = predict_multimomial(model, int_notes, predict_len = 100, temperature = 2.2)\n", " generated_seq_multinomial = [int_to_note[e] for e in generated_seq_multinomial]\n", " pred_midi_multinomial = create_midi(generated_seq_multinomial)\n", " \n", " pred_midi_multinomial.write('midi', fp='result.midi')\n", " \n", " # sound_font = \"/usr/share/sounds/sf2/FluidR3_GM.sf2\"\n", " FluidSynth().midi_to_audio('result.midi', 'result.wav')\n", " return 'result.wav', 'result.midi'" ] }, { "cell_type": "code", "execution_count": 115, "id": "7227d54b", "metadata": {}, "outputs": [ { "ename": "FileNotFoundError", "evalue": "[Errno 2] No such file or directory: 'fluidsynth'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", "Input \u001b[0;32mIn [115]\u001b[0m, in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m example_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/Users/dmytrolopushanskyy/Documents/ucu/music-generation/examples/mozart.midi\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mprocess_input\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexample_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n", "Input \u001b[0;32mIn [114]\u001b[0m, in \u001b[0;36mprocess_input\u001b[0;34m(input_midi_file, input_randomness, input_duration)\u001b[0m\n\u001b[1;32m 10\u001b[0m pred_midi_multinomial\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmidi\u001b[39m\u001b[38;5;124m'\u001b[39m, fp\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mresult.midi\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 12\u001b[0m sound_font \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/usr/share/sounds/sf2/FluidR3_GM.sf2\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 13\u001b[0m \u001b[43mFluidSynth\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmidi_to_audio\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mresult.midi\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mresult.wav\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mresult.wav\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mresult.midi\u001b[39m\u001b[38;5;124m'\u001b[39m\n", "File \u001b[0;32m~/miniconda3/envs/hugface-demo/lib/python3.9/site-packages/midi2audio.py:46\u001b[0m, in \u001b[0;36mFluidSynth.midi_to_audio\u001b[0;34m(self, midi_file, audio_file)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmidi_to_audio\u001b[39m(\u001b[38;5;28mself\u001b[39m, midi_file, audio_file):\n\u001b[0;32m---> 46\u001b[0m \u001b[43msubprocess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mfluidsynth\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m-ni\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msound_font\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmidi_file\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m-F\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maudio_file\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m-r\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msample_rate\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/miniconda3/envs/hugface-demo/lib/python3.9/subprocess.py:349\u001b[0m, in \u001b[0;36mcall\u001b[0;34m(timeout, *popenargs, **kwargs)\u001b[0m\n\u001b[1;32m 341\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcall\u001b[39m(\u001b[38;5;241m*\u001b[39mpopenargs, timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 342\u001b[0m \u001b[38;5;124;03m\"\"\"Run command with arguments. Wait for command to complete or\u001b[39;00m\n\u001b[1;32m 343\u001b[0m \u001b[38;5;124;03m timeout, then return the returncode attribute.\u001b[39;00m\n\u001b[1;32m 344\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[38;5;124;03m retcode = call([\"ls\", \"-l\"])\u001b[39;00m\n\u001b[1;32m 348\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 349\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mPopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mpopenargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m p:\n\u001b[1;32m 350\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 351\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m p\u001b[38;5;241m.\u001b[39mwait(timeout\u001b[38;5;241m=\u001b[39mtimeout)\n", "File \u001b[0;32m~/miniconda3/envs/hugface-demo/lib/python3.9/subprocess.py:951\u001b[0m, in \u001b[0;36mPopen.__init__\u001b[0;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, user, group, extra_groups, encoding, errors, text, umask)\u001b[0m\n\u001b[1;32m 947\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtext_mode:\n\u001b[1;32m 948\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstderr \u001b[38;5;241m=\u001b[39m io\u001b[38;5;241m.\u001b[39mTextIOWrapper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstderr,\n\u001b[1;32m 949\u001b[0m encoding\u001b[38;5;241m=\u001b[39mencoding, errors\u001b[38;5;241m=\u001b[39merrors)\n\u001b[0;32m--> 951\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execute_child\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexecutable\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpreexec_fn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mclose_fds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 952\u001b[0m \u001b[43m \u001b[49m\u001b[43mpass_fds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcwd\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43menv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 953\u001b[0m \u001b[43m \u001b[49m\u001b[43mstartupinfo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreationflags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshell\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 954\u001b[0m \u001b[43m \u001b[49m\u001b[43mp2cread\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp2cwrite\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 955\u001b[0m \u001b[43m \u001b[49m\u001b[43mc2pread\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mc2pwrite\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 956\u001b[0m \u001b[43m \u001b[49m\u001b[43merrread\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merrwrite\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 957\u001b[0m \u001b[43m \u001b[49m\u001b[43mrestore_signals\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 958\u001b[0m \u001b[43m \u001b[49m\u001b[43mgid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mumask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 959\u001b[0m \u001b[43m \u001b[49m\u001b[43mstart_new_session\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 960\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m 961\u001b[0m \u001b[38;5;66;03m# Cleanup if the child failed starting.\u001b[39;00m\n\u001b[1;32m 962\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mfilter\u001b[39m(\u001b[38;5;28;01mNone\u001b[39;00m, (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstdin, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstdout, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstderr)):\n", "File \u001b[0;32m~/miniconda3/envs/hugface-demo/lib/python3.9/subprocess.py:1821\u001b[0m, in \u001b[0;36mPopen._execute_child\u001b[0;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, gid, gids, uid, umask, start_new_session)\u001b[0m\n\u001b[1;32m 1819\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m errno_num \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 1820\u001b[0m err_msg \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mstrerror(errno_num)\n\u001b[0;32m-> 1821\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m child_exception_type(errno_num, err_msg, err_filename)\n\u001b[1;32m 1822\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m child_exception_type(err_msg)\n", "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'fluidsynth'" ] } ], "source": [ "example_path = '/Users/dmytrolopushanskyy/Documents/ucu/music-generation/examples/mozart.midi'\n", "process_input(example_path, 2, 3)" ] }, { "cell_type": "code", "execution_count": null, "id": "01491a9f", "metadata": {}, "outputs": [], "source": [ "midi_file_desc = \"\"\"\n", "This model allows to generate music based on your input. \n", "Please upload a MIDI file below, choose music randomness and duration. Enjoy!\n", "\"\"\"\n", "\n", "article = \"\"\"# Music Generation\n", "This project has been created by the students of Ukrainian Catholic University for our ML course.\n", "\n", "We are using a GRU model to output new notes based on the given input. You can find more information at our Git repo: https://github.com/DmytroLopushanskyy/music-generation\n", "We are using a language model to create music by treating a musical standard MIDI a simple text, with tokens for note values, note duration, and separations to denote movement forward in time.\n", "\"\"\"\n", "\n", "iface = gr.Interface(\n", " fn=process_input, \n", " inputs=[\n", " gr.inputs.File(label=midi_file_desc),\n", " gr.inputs.Slider(0, 250, default=100, step=50),\n", " gr.inputs.Radio([10, 20, 30], type=\"value\", default=20)\n", " ], \n", " outputs=[\"audio\", \"file\"],\n", " article=article,\n", " examples=['examples/mozart.midi']\n", ")\n", "\n", "iface.launch()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.12" } }, "nbformat": 4, "nbformat_minor": 5 }