Update app.py

app.py

@@ -33,8 +33,7 @@ import matplotlib.pyplot as plt
 @spaces.GPU
 def Harmonize_Melody(input_src_midi,
                     source_melody_transpose_value,
-                    harmonizer_melody_chunk_size,
-                    harmonizer_max_matches_count,
+                    model_top_k_sampling_value,
                     melody_MIDI_patch_number,
                     harmonized_accompaniment_MIDI_patch_number,
                     base_MIDI_patch_number
@@ -48,13 +47,13 @@ def Harmonize_Melody(input_src_midi,
 
     sfn = os.path.basename(input_src_midi.name)
     sfn1 = sfn.split('.')[0]
+    
     print('Input src MIDI name:', sfn)
 
     print('=' * 70)
     print('Requested settings:')
     print('Source melody transpose value:', source_melody_transpose_value)
-    print('Harmonizer melody chunk size:', harmonizer_melody_chunk_size)
-    print('Harmonizer max matrches count:', harmonizer_max_matches_count)
+    print('Model top_k sampling value:', model_top_k_sampling_value)
     print('Melody MIDI patch number:', melody_MIDI_patch_number)
     print('Harmonized accompaniment MIDI patch number:', harmonized_accompaniment_MIDI_patch_number)
     print('Base MIDI patch number:', base_MIDI_patch_number)
@@ -95,32 +94,6 @@ def Harmonize_Melody(input_src_midi,
     print('Melody has', len(mel_pitches), 'notes')
     print('=' * 70)
 
-    #==================================================================
-    
-    print('=' * 70)
-    print('Creating chords dict...')
-    print('=' * 70)
-
-    chords_groups = []
-    
-    for i in range(12):
-      grp = []
-      for c in TMIDIX.ALL_CHORDS_FILTERED:
-        if i in c:
-          grp.append(c)
-    
-      if grp:
-        chords_groups.append(grp)
-    
-    max_grp_len = len(max(chords_groups, key=len))
-    
-    chords_groups_padded = []
-    
-    for c in chords_groups:
-      grp = c + [[-1]] * (max_grp_len-len(c))
-    
-      chords_groups_padded.extend(grp)
-
     #===============================================================================
 
     print('=' * 70)
@@ -129,87 +102,80 @@ def Harmonize_Melody(input_src_midi,
 
     print('Loading Melody Harmonizer Transformer Model...')
 
+    SEQ_LEN = 75
+    PAD_IDX = 144
     
-
-    print('=' * 70)
-    print('Harmonizing...')
-    print('=' * 70)
-
-    #===============================================================================
+    # instantiate the model
     
-    song = []
+    model = TransformerWrapper(
+        num_tokens = PAD_IDX+1,
+        max_seq_len = SEQ_LEN,
+        attn_layers = Decoder(dim = 1024, depth = 12, heads = 16, attn_flash = True)
+        )
     
-    csize = harmonizer_melody_chunk_size
-    matches_mem_size = harmonizer_max_matches_count
+    model = AutoregressiveWrapper(model, ignore_index = PAD_IDX, pad_value=PAD_IDX)
     
-    i = 0
-    dev = 0
-    dchunk = []
+    model_path = 'Melody_Harmonizer_Transformer_Trained_Model_7522_steps_0.6545_loss_0.7906_acc.pth'
     
-    #===============================================================================
-
-    def find_best_match(matches):
+    model.load_state_dict(torch.load(model_path))
     
-      mlens = []
+    model.cuda()
     
-      for sidx in matches:
-        mlen = len(TMIDIX.flatten(long_chords_chunks_mult[sidx[0]][sidx[1]:sidx[1]+(csize // 2)]))
-        mlens.append(mlen)
+    dtype = torch.bfloat16
     
-      max_len = max(mlens)
-      max_len_idx = mlens.index(max_len)
+    ctx = torch.amp.autocast(device_type='cuda', dtype=dtype)
     
-      return matches[max_len_idx]
+    model.eval()
+    
+    print('Done!')    
+
+    print('=' * 70)
+    print('Harmonizing...')
+    print('=' * 70)
 
     #===============================================================================
     
-    while i < len(mel_pitches):
+    mel_remainder_value = (((len(mel_pitches) // 24)+1) * 24) - len(mel_pitches)
     
-      matches = []
+    mel_pitches_ext = mel_pitches + mel_pitches[:mel_remainder_value]
     
-      for midx, mel in enumerate(long_mels_chunks_mult):
-        if len(mel) >= csize:
-          schunk = mel_pitches[i:i+csize]
-          idx = HaystackSearch.HaystackSearch(schunk, mel)
+    song = []
     
-          if idx != -1:
-            matches.append([midx, idx])
-            if matches_mem_size > -1:
-              if len(matches) > matches_mem_size:
-                break
+    for i in range(0, len(mel_pitches_ext)-12, 12):
     
-      if matches:
+      mel_chunk = mel_pitches_ext[i:i+24]
     
-        sidx = find_best_match(matches)
+      data = [141] + mel_chunk + [142]
     
-        fchunk = long_chords_chunks_mult[sidx[0]][sidx[1]:sidx[1]+csize]
+      for j in range(24):
     
-        song.extend(fchunk[:(csize // 2)])
-        i += (csize // 2)
-        dchunk = fchunk
-        dev = 0
-        print('step', i)
-        
-      else:
+        data.append(mel_chunk[j])
+    
+        x = torch.tensor([data], dtype=torch.long, device='cuda')
     
-        if dchunk:
+        with ctx:
+          out = model.generate(x,
+                              1,
+                              filter_logits_fn=top_k,
+                              filter_kwargs={'k': model_top_k_sampling_value},
+                              temperature=0.9,
+                              return_prime=False,
+                              verbose=False)
     
-          song.append(dchunk[(csize // 2)+dev])
-          dev += 1
-          i += 1
-          print('dead chord', i, dev)
-        else:
-          print('DEAD END!!!')
-          song.append([mel_pitches[0]+48])
-          break
+        outy = out.tolist()[0]
     
+        data.append(outy[0])
     
-        if dev == csize // 2:
-          print('DEAD END!!!')
-          break
+      if i != len(mel_pitches_ext)-24:
     
-    song = song[:len(mel_pitches)]
+        song.extend(data[26:50])
+      else:
+        song.extend(data[26:])
     
+    song = song[:len(mel_pitches) * 2]
+    
+    #===============================================================================
+
     print('Harmonized', len(song), 'out of', len(mel_pitches), 'notes')
 
     print('Done!')
@@ -233,22 +199,28 @@ def Harmonize_Melody(input_src_midi,
     patches[3] = melody_MIDI_patch_number
     
     for i, s in enumerate(song):
-    
-      time = mel_score[i][1] * 16
-      dur = mel_score[i][2] * 16
 
-      output_score.append(['note', time, dur, 3,  mel_score[i][4], 115+(mel_score[i][4] % 12), 40])
+        if 11 < s < 141:
     
-      for p in s:
-        output_score.append(['note', time, dur, 0, p, max(40, p), harmonized_accompaniment_MIDI_patch_number])
-                    
-      if base_MIDI_patch_number > -1:
-        output_score.append(['note', time, dur, 2, (s[-1] %  12)+24, 120-(s[-1] %  12), base_MIDI_patch_number])
+            time = mel_score[i][1] * 16
+            dur = mel_score[i][2] * 16
+            
+            output_score.append(['note', time, dur, 3,  mel_score[i][4], 115+(mel_score[i][4] % 12), 40])
+
+            chord = TMIDIX.ALL_CHORDS_FILTERED[s-12]
+            
+            for c in chord:
+                
+                pitch = 48+c
+                output_score.append(['note', time, dur, 0, pitch, max(40, pitch), harmonized_accompaniment_MIDI_patch_number])
+                        
+                if base_MIDI_patch_number > -1:
+                    output_score.append(['note', time, dur, 2, chord[-1]+24, 120-chord[-1], base_MIDI_patch_number])
 
-    fn1 = "Monophonic-MIDI-Melody-Harmonizer-Composition"
+    fn1 = "Melody-Harmonizer-Transformer-Composition"
     
     detailed_stats = TMIDIX.Tegridy_ms_SONG_to_MIDI_Converter(output_score,
-                                                              output_signature = 'Monophonic MIDI Melody Harmonizer',
+                                                              output_signature = 'Melody Harmonizer Transformer',
                                                               output_file_name = fn1,
                                                               track_name='Project Los Angeles',
                                                               list_of_MIDI_patches=patches
@@ -346,8 +318,7 @@ if __name__ == "__main__":
         gr.Markdown("## Select harmonization options")
 
         source_melody_transpose_value = gr.Slider(-6, 6, value=0, step=1, label="Source melody transpose value", info="You can transpose source melody by specified number of semitones if the original melody key does not harmonize well")
-        harmonizer_melody_chunk_size = gr.Slider(4, 16, value=8, step=2, label="Hamonizer melody chunk size", info="Larger chunk sizes result in better harmonization at the cost of speed and harminzation length")
-        harmonizer_max_matches_count = gr.Slider(-1, 20, value=0, step=1, label="Harmonizer max matches count", info="Maximum number of harmonized chords per melody note to collect and to select from")
+        model_top_k_sampling_value = gr.Slider(1, 50, value=15, step=1, label="Model sampling top_k value", info="Decreasing this value may produce better harmonization results in some cases")
        
         melody_MIDI_patch_number = gr.Slider(0, 127, value=40, step=1, label="Source melody MIDI patch number")
         harmonized_accompaniment_MIDI_patch_number = gr.Slider(0, 127, value=0, step=1, label="Harmonized accompaniment MIDI patch number")
@@ -370,8 +341,7 @@ if __name__ == "__main__":
         run_event = run_btn.click(Harmonize_Melody, 
                                                   [input_src_midi,
                                                     source_melody_transpose_value,
-                                                    harmonizer_melody_chunk_size,
-                                                    harmonizer_max_matches_count,
+                                                    model_top_k_sampling_value,
                                                     melody_MIDI_patch_number,
                                                     harmonized_accompaniment_MIDI_patch_number,
                                                     base_MIDI_patch_number],                                                                                                                       
@@ -380,18 +350,17 @@ if __name__ == "__main__":
 
         gr.Examples(
             [
-            ["USSR Anthem Seed Melody.mid", 0, 12, -1, 40, 0, 35],
+            ["USSR Anthem Seed Melody.mid", 0, 15, 40, 0, 35],
             ],
             [input_src_midi,
             source_melody_transpose_value,
-            harmonizer_melody_chunk_size,
-            harmonizer_max_matches_count,
+            model_top_k_sampling_value,
             melody_MIDI_patch_number,
             harmonized_accompaniment_MIDI_patch_number,
             base_MIDI_patch_number],                                                                                                                       
             [output_audio, output_plot, output_midi, output_summary],
             Harmonize_Melody,
-            cache_examples=False,
+            cache_examples=True,
         )
    
         app.queue().launch()