skytnt commited on
Commit
6373391
1 Parent(s): f60d1e9

update midi visualizer

Browse files
Files changed (2) hide show
  1. app.py +1 -1
  2. javascript/app.js +162 -27
app.py CHANGED
@@ -281,7 +281,7 @@ if __name__ == "__main__":
281
  stop_btn = gr.Button("stop and output")
282
  output_midi_seq = gr.Variable()
283
  output_midi_visualizer = gr.HTML(elem_id="midi_visualizer_container")
284
- output_audio = gr.Audio(label="output audio", format="mp3")
285
  output_midi = gr.File(label="output midi", file_types=[".mid"])
286
  run_event = run_btn.click(run, [input_model, tab_select, input_instruments, input_drum_kit, input_midi,
287
  input_midi_events, input_gen_events, input_temp, input_top_p, input_top_k,
 
281
  stop_btn = gr.Button("stop and output")
282
  output_midi_seq = gr.Variable()
283
  output_midi_visualizer = gr.HTML(elem_id="midi_visualizer_container")
284
+ output_audio = gr.Audio(label="output audio", format="mp3", elem_id="midi_audio")
285
  output_midi = gr.File(label="output midi", file_types=[".mid"])
286
  run_event = run_btn.click(run, [input_model, tab_select, input_instruments, input_drum_kit, input_midi,
287
  input_midi_events, input_gen_events, input_temp, input_top_p, input_top_k,
javascript/app.js CHANGED
@@ -58,12 +58,36 @@ document.addEventListener("DOMContentLoaded", function() {
58
  mse_receiver_inited = msg_receiver;
59
  }
60
  })
61
- })();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
 
63
  class MidiVisualizer extends HTMLElement{
64
  constructor() {
65
  super();
66
  this.midiEvents = [];
 
 
67
  this.wrapper = null;
68
  this.svg = null;
69
  this.timeLine = null;
@@ -71,10 +95,14 @@ class MidiVisualizer extends HTMLElement{
71
  noteHeight : 4,
72
  beatWidth: 32
73
  }
 
74
  this.svgWidth = 0;
75
  this.t1 = 0;
76
  this.playTime = 0
 
77
  this.colorMap = new Map();
 
 
78
  this.init();
79
  }
80
 
@@ -101,48 +129,46 @@ class MidiVisualizer extends HTMLElement{
101
  this.setPlayTime(0);
102
  }
103
 
104
- setPlayTime(t){
105
- this.playTime = t
106
- let x = Math.round((t/16)*this.config.beatWidth)
107
- this.timeLine.setAttribute('x1', `${x}`);
108
- this.timeLine.setAttribute('y1', '0');
109
- this.timeLine.setAttribute('x2', `${x}`);
110
- this.timeLine.setAttribute('y2', `${this.config.noteHeight*128}`);
111
- }
112
-
113
  clearMidiEvents(){
 
114
  this.midiEvents = [];
115
- this.svgWidth = 0
116
- this.svg.innerHTML = ''
117
  this.t1 = 0
118
  this.colorMap.clear()
119
- this.svg.style.width = `${this.svgWidth}px`;
120
  this.setPlayTime(0);
 
 
 
 
121
  this.svg.appendChild(this.timeLine)
122
  }
123
 
124
  appendMidiEvent(midiEvent){
125
  if(midiEvent instanceof Array && midiEvent.length > 0){
126
- this.midiEvents.push(midiEvent);
127
  this.t1 += midiEvent[1]
128
- let t = this.t1*16 + midiEvent[2]
 
129
  if(midiEvent[0] === "note"){
130
- let track = midiEvent[3]
131
- let duration = midiEvent[4]
132
- let channel = midiEvent[5]
133
- let pitch = midiEvent[6]
134
- let velocity = midiEvent[7]
135
- let x = (t/16)*this.config.beatWidth
136
  let y = (127 - pitch)*this.config.noteHeight
137
- let w = (duration/16)*this.config.beatWidth
138
  let h = this.config.noteHeight
139
  this.svgWidth = Math.ceil(Math.max(x + w, this.svgWidth))
140
  let color = this.getColor(track, channel)
141
  let opacity = Math.min(1, velocity/127 + 0.1).toFixed(2)
142
- this.drawNote(x,y,w,h, `rgba(${color[0]}, ${color[1]}, ${color[2]}, ${opacity})`)
 
143
  this.setPlayTime(t);
144
  this.wrapper.scrollTo(this.svgWidth - this.wrapper.offsetWidth, 0)
145
  }
 
146
  this.svg.style.width = `${this.svgWidth}px`;
147
  }
148
 
@@ -154,7 +180,7 @@ class MidiVisualizer extends HTMLElement{
154
  if(!!color){
155
  return color;
156
  }
157
- color = [Math.round(Math.random()*240) + 10, Math.round(Math.random()*240)+ 10, Math.round(Math.random()*240)+ 10];
158
  this.colorMap.set(key, color);
159
  return color;
160
  }
@@ -173,13 +199,116 @@ class MidiVisualizer extends HTMLElement{
173
  rect.setAttribute('height', `${Math.round(h)}`);
174
  this.svg.appendChild(rect);
175
  return rect
176
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
  }
178
 
179
  customElements.define('midi-visualizer', MidiVisualizer);
180
 
181
  (()=>{
182
  let midi_visualizer_container_inited = null
 
183
  let midi_visualizer = document.createElement('midi-visualizer')
184
  onUiUpdate((m)=>{
185
  let app = gradioApp()
@@ -188,6 +317,11 @@ customElements.define('midi-visualizer', MidiVisualizer);
188
  midi_visualizer_container.appendChild(midi_visualizer)
189
  midi_visualizer_container_inited = midi_visualizer_container;
190
  }
 
 
 
 
 
191
  })
192
 
193
  function createProgressBar(progressbarContainer){
@@ -243,8 +377,9 @@ customElements.define('midi-visualizer', MidiVisualizer);
243
  setProgressBar(midi_visualizer_container_inited, progress, total)
244
  break;
245
  case "visualizer_end":
246
- midi_visualizer.setPlayTime(0)
247
- removeProgressBar(midi_visualizer_container_inited)
 
248
  break;
249
  default:
250
  }
 
58
  mse_receiver_inited = msg_receiver;
59
  }
60
  })
61
+ })()
62
+
63
+ function HSVtoRGB(h, s, v) {
64
+ let r, g, b, i, f, p, q, t;
65
+ i = Math.floor(h * 6);
66
+ f = h * 6 - i;
67
+ p = v * (1 - s);
68
+ q = v * (1 - f * s);
69
+ t = v * (1 - (1 - f) * s);
70
+ switch (i % 6) {
71
+ case 0: r = v; g = t; b = p; break;
72
+ case 1: r = q; g = v; b = p; break;
73
+ case 2: r = p; g = v; b = t; break;
74
+ case 3: r = p; g = q; b = v; break;
75
+ case 4: r = t; g = p; b = v; break;
76
+ case 5: r = v; g = p; b = q; break;
77
+ }
78
+ return {
79
+ r: Math.round(r * 255),
80
+ g: Math.round(g * 255),
81
+ b: Math.round(b * 255)
82
+ };
83
+ }
84
 
85
  class MidiVisualizer extends HTMLElement{
86
  constructor() {
87
  super();
88
  this.midiEvents = [];
89
+ this.activeNotes = [];
90
+ this.midiTimes = [];
91
  this.wrapper = null;
92
  this.svg = null;
93
  this.timeLine = null;
 
95
  noteHeight : 4,
96
  beatWidth: 32
97
  }
98
+ this.timePreBeat = 16
99
  this.svgWidth = 0;
100
  this.t1 = 0;
101
  this.playTime = 0
102
+ this.playTimeMs = 0
103
  this.colorMap = new Map();
104
+ this.playing = false;
105
+ this.timer = null;
106
  this.init();
107
  }
108
 
 
129
  this.setPlayTime(0);
130
  }
131
 
 
 
 
 
 
 
 
 
 
132
  clearMidiEvents(){
133
+ this.pause()
134
  this.midiEvents = [];
135
+ this.activeNotes = [];
136
+ this.midiTimes = [];
137
  this.t1 = 0
138
  this.colorMap.clear()
 
139
  this.setPlayTime(0);
140
+ this.playTimeMs = 0
141
+ this.svgWidth = 0
142
+ this.svg.innerHTML = ''
143
+ this.svg.style.width = `${this.svgWidth}px`;
144
  this.svg.appendChild(this.timeLine)
145
  }
146
 
147
  appendMidiEvent(midiEvent){
148
  if(midiEvent instanceof Array && midiEvent.length > 0){
149
+
150
  this.t1 += midiEvent[1]
151
+ let t = this.t1*this.timePreBeat + midiEvent[2]
152
+ midiEvent = [midiEvent[0], t].concat(midiEvent.slice(3))
153
  if(midiEvent[0] === "note"){
154
+ let track = midiEvent[2]
155
+ let duration = midiEvent[3]
156
+ let channel = midiEvent[4]
157
+ let pitch = midiEvent[5]
158
+ let velocity = midiEvent[6]
159
+ let x = (t/this.timePreBeat)*this.config.beatWidth
160
  let y = (127 - pitch)*this.config.noteHeight
161
+ let w = (duration/this.timePreBeat)*this.config.beatWidth
162
  let h = this.config.noteHeight
163
  this.svgWidth = Math.ceil(Math.max(x + w, this.svgWidth))
164
  let color = this.getColor(track, channel)
165
  let opacity = Math.min(1, velocity/127 + 0.1).toFixed(2)
166
+ let rect = this.drawNote(x,y,w,h, `rgba(${color.r}, ${color.g}, ${color.b}, ${opacity})`)
167
+ midiEvent.push(rect)
168
  this.setPlayTime(t);
169
  this.wrapper.scrollTo(this.svgWidth - this.wrapper.offsetWidth, 0)
170
  }
171
+ this.midiEvents.push(midiEvent);
172
  this.svg.style.width = `${this.svgWidth}px`;
173
  }
174
 
 
180
  if(!!color){
181
  return color;
182
  }
183
+ color = HSVtoRGB(Math.random(),Math.random()*0.5 + 0.5,1);
184
  this.colorMap.set(key, color);
185
  return color;
186
  }
 
199
  rect.setAttribute('height', `${Math.round(h)}`);
200
  this.svg.appendChild(rect);
201
  return rect
202
+ }
203
+
204
+ finishAppendMidiEvent(){
205
+ let midiEvents = this.midiEvents.sort((a, b)=>a[1]-b[1])
206
+ let tempo = (60 / 120) * 10 ** 3
207
+ let ms = 0
208
+ let lastT = 0
209
+ this.midiTimes.push({ms:ms, t: 0, tempo: tempo})
210
+ midiEvents.forEach((midiEvent)=>{
211
+ let t = midiEvent[1]
212
+ ms += ((t- lastT) / this.timePreBeat) * tempo
213
+ if(midiEvent[0]==="set_tempo"){
214
+ tempo = (60 / midiEvent[3]) * 10 ** 3
215
+ this.midiTimes.push({ms:ms, t: t, tempo: tempo})
216
+ }
217
+ lastT = t
218
+ })
219
+ }
220
+
221
+ setPlayTime(t){
222
+ this.playTime = t
223
+ let x = Math.round((t/this.timePreBeat)*this.config.beatWidth)
224
+ this.timeLine.setAttribute('x1', `${x}`);
225
+ this.timeLine.setAttribute('y1', '0');
226
+ this.timeLine.setAttribute('x2', `${x}`);
227
+ this.timeLine.setAttribute('y2', `${this.config.noteHeight*128}`);
228
+
229
+ this.wrapper.scrollTo(Math.max(0, x - this.wrapper.offsetWidth/2), 0)
230
+
231
+ if(this.playing){
232
+ let activeNotes = []
233
+ this.removeActiveNotes(this.activeNotes)
234
+ this.midiEvents.forEach((midiEvent)=>{
235
+ if(midiEvent[0] === "note"){
236
+ let time = midiEvent[1]
237
+ let duration = midiEvent[3]
238
+ let note = midiEvent[midiEvent.length - 1]
239
+ if(time <=this.playTime && time+duration>= this.playTime){
240
+ activeNotes.push(note)
241
+ }
242
+ }
243
+ })
244
+ this.addActiveNotes(activeNotes)
245
+ }
246
+ }
247
+
248
+ setPlayTimeMs(ms){
249
+ this.playTimeMs = ms
250
+ let playTime = 0
251
+ for(let i =0;i<this.midiTimes.length;i++){
252
+ let midiTime = this.midiTimes[i]
253
+ if(midiTime.ms>=ms){
254
+ break;
255
+ }
256
+ playTime = midiTime.t + (ms-midiTime.ms) * this.timePreBeat / midiTime.tempo
257
+ }
258
+ this.setPlayTime(playTime)
259
+ }
260
+
261
+ addActiveNotes(notes){
262
+ notes.forEach((note)=>{
263
+ this.activeNotes.push(note)
264
+ note.classList.add('active');
265
+ });
266
+ }
267
+
268
+ removeActiveNotes(notes){
269
+ notes.forEach((note)=>{
270
+ let idx = this.activeNotes.indexOf(note)
271
+ if(idx>-1)
272
+ this.activeNotes.splice(idx, 1);
273
+ note.classList.remove('active');
274
+ });
275
+ }
276
+
277
+ play(){
278
+ this.playing = true;
279
+ this.timer = setInterval(() => {
280
+ this.setPlayTimeMs(this.playTimeMs + 10)
281
+ }, 10);
282
+ }
283
+
284
+ pause(){
285
+ if(!!this.timer)
286
+ clearInterval(this.timer)
287
+ this.removeActiveNotes(this.activeNotes)
288
+ this.timer = null;
289
+ this.playing = false;
290
+ }
291
+
292
+
293
+ bindAudioPlayer(audio){
294
+ this.pause()
295
+ audio.addEventListener("play", (event)=>{
296
+ this.play()
297
+ })
298
+ audio.addEventListener("pause", (event)=>{
299
+ this.pause()
300
+ })
301
+ audio.addEventListener("timeupdate", (event)=>{
302
+ this.setPlayTimeMs(event.target.currentTime*10**3)
303
+ })
304
+ }
305
  }
306
 
307
  customElements.define('midi-visualizer', MidiVisualizer);
308
 
309
  (()=>{
310
  let midi_visualizer_container_inited = null
311
+ let midi_audio_inited = null;
312
  let midi_visualizer = document.createElement('midi-visualizer')
313
  onUiUpdate((m)=>{
314
  let app = gradioApp()
 
317
  midi_visualizer_container.appendChild(midi_visualizer)
318
  midi_visualizer_container_inited = midi_visualizer_container;
319
  }
320
+ let midi_audio = app.querySelector("#midi_audio > audio");
321
+ if(!!midi_audio && midi_audio_inited!==midi_audio){
322
+ midi_visualizer.bindAudioPlayer(midi_audio)
323
+ midi_audio_inited = midi_audio
324
+ }
325
  })
326
 
327
  function createProgressBar(progressbarContainer){
 
377
  setProgressBar(midi_visualizer_container_inited, progress, total)
378
  break;
379
  case "visualizer_end":
380
+ midi_visualizer.finishAppendMidiEvent()
381
+ midi_visualizer.setPlayTime(0);
382
+ removeProgressBar(midi_visualizer_container_inited);
383
  break;
384
  default:
385
  }