Hugging Face's logo

Spaces:
yiwv
/
music-gen-kit
Runtime error

App Files Community
yiwv commited on
Commit
fde0931
•
1 Parent(s): 9e10c76

wip: use model in magenta html

Browse files
Files changed (2) hide show
  1. index.html +146 -0
  2. script.py +13 -0
index.html ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>Magenta.js Model Loader and Player</title>
7
+ <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@4.11.0"></script>
8
+
9
+ <script src="https://cdn.jsdelivr.net/npm/@magenta/music"></script>
10
+ </head>
11
+ <body>
12
+
13
+ <button onclick="generateAndPlayMusic()">Generate and Play Music</button>
14
+
15
+ <script>
16
+ class MSEWithPositivePressure extends tf.layers.Layer {
17
+ constructor() {
18
+ super({});
19
+ }
20
+
21
+ call(inputs) {
22
+ let y_true = inputs[0];
23
+ let y_pred = inputs[1];
24
+ let mse = tf.mean(tf.square(tf.sub(y_true, y_pred)));
25
+ let positive_pressure = tf.mean(tf.maximum(tf.scalar(0), tf.neg(y_pred)));
26
+ return tf.add(mse, positive_pressure);
27
+ }
28
+
29
+ static get className() {
30
+ return 'MSEWithPositivePressure';
31
+ }
32
+ }
33
+
34
+ // tf.serialization.registerClass(MSEWithPositivePressure);
35
+
36
+ let model;
37
+
38
+ async function loadModel() {
39
+ tf.serialization.registerClass(MSEWithPositivePressure);
40
+
41
+ // model = await tf.loadLayersModel('/js_model/model.json');
42
+ model = await tf.loadGraphModel('/js_model/model.json');
43
+
44
+ console.log("Model loaded successfully!");
45
+ }
46
+
47
+
48
+ async function generateAndPlayMusic() {
49
+ if (!model) {
50
+ await loadModel();
51
+ }
52
+
53
+ let inputValues = [60, 0.5, 0.5, 62, 0.5, 0.5, 64, 0.5, 0.5];
54
+ let numNotes = inputValues.length / 3;
55
+ let inputSequence;
56
+ if (numNotes > 25) {
57
+ let inputData = new Array(numNotes * 3).fill(0).concat(inputValues);
58
+ inputSequence = tf.tensor3d(inputData, [1, numNotes, 3]);
59
+ } else {
60
+ const padding = new Array((25 - numNotes) * 3).fill(0);
61
+ let inputData = padding.concat(inputValues);
62
+ inputSequence = tf.tensor3d(inputData, [1, 25, 3]);
63
+ }
64
+
65
+ // inputSequence = inputSequence.bufferSync();
66
+ // for (let i = 0; i < inputValues.length; i++) {
67
+ // inputSequence.set(inputValues[i], 0, 24 - numNotes + Math.floor(i / 3), i % 3);
68
+ // }
69
+ // inputSequence = inputSequence.toTensor();
70
+
71
+ const temperature = 2.0 // 0.5 // 2.0;
72
+ const numPredictions = 40; // 120;
73
+ let generatedNotes = [];
74
+
75
+ for (let i = 0; i < numPredictions; i++) {
76
+ const predictions = await model.executeAsync(inputSequence);
77
+
78
+ // const pitchProbs = tf.softmax(predictions[2]);
79
+ // const pitch = tf.multinomial(pitchProbs, 1).dataSync()[0];
80
+
81
+ // const pitchProbs = tf.softmax(predictions[2].dataSync()).div(temperature);
82
+ // const pitch = tf.multinomial(pitchProbs, 1).dataSync()[0];
83
+
84
+ // const pitchLogitsArray = predictions[2].dataSync();
85
+ const pitchLogitsArray = predictions[2].dataSync().map(value => value / temperature);
86
+
87
+ // const pitchLogitsTensor = tf.tensor(pitchLogitsArray).div(temperature);
88
+ // const pitchProbs = tf.softmax(pitchLogitsTensor);
89
+ const pitchProbs = tf.softmax(pitchLogitsArray);
90
+ const pitch = tf.multinomial(pitchProbs, 1).dataSync()[0];
91
+
92
+ const clippedPitch = Math.min(Math.max(pitch, 21), 108);
93
+
94
+ const step = Math.max(0, predictions[1].dataSync()[0]);
95
+ const duration = Math.max(0, predictions[0].dataSync()[0]);
96
+
97
+
98
+ // console.log('///////////////////')
99
+ // console.log(predictions[0].dataSync(),predictions[1].dataSync(),predictions[2].dataSync())
100
+ // console.log('/////////////////// //')
101
+
102
+ // console.log({predictions, pitch, step, duration})
103
+
104
+ console.log('pitch:', pitch, {pitchLogitsArray, pitchProbs})
105
+
106
+ generatedNotes.push([clippedPitch, step, duration]);
107
+
108
+
109
+ // 新しいノートを生成
110
+ const newNote = tf.tensor3d([[[clippedPitch * 1.0, step, duration]]], [1, 1, 3]);
111
+
112
+ // 入力シーケンスに新しいノートを追加
113
+ inputSequence = inputSequence.slice([0, 1, 0], [-1, -1, -1]).concat(newNote, 1);
114
+
115
+ }
116
+
117
+ // 生成されたノートをNoteSequenceに変換
118
+ const noteSequence = {
119
+ ticksPerQuarter: 220,
120
+ totalTime: generatedNotes.length / 2,
121
+ timeSignatures: [{ time: 0, numerator: 4, denominator: 4 }],
122
+ tempos: [{ time: 0, qpm: 120 }],
123
+ notes: generatedNotes.map((note, index) => ({
124
+ startTime: index / 2,
125
+ endTime: (index + 1) / 2,
126
+ pitch: note[0],
127
+ velocity: 80
128
+ }))
129
+ };
130
+
131
+ // // NoteSequenceを再生する
132
+ // const player = new mm.Player();
133
+ // player.start(noteSequence);
134
+
135
+
136
+ // Play the note sequence using SoundFontPlayer
137
+ const soundfontURL = 'https://storage.googleapis.com/magentadata/js/soundfonts/sgm_plus';
138
+ const player = new mm.SoundFontPlayer(soundfontURL);
139
+ player.start(noteSequence);
140
+ }
141
+
142
+
143
+ </script>
144
+
145
+ </body>
146
+ </html>
script.py CHANGED
@@ -20,10 +20,19 @@ def predict_next_note(notes, keras_model, temperature=1.0):
20
  assert temperature > 0
21
  inputs = tf.expand_dims(notes, 0)
22
  predictions = model.predict(inputs)
 
 
23
  pitch_logits = predictions['pitch']
24
  step = predictions['step']
25
  duration = predictions['duration']
26
 
 
 
 
 
 
 
 
27
  pitch_logits /= temperature
28
  pitch = tf.random.categorical(pitch_logits, num_samples=1)
29
  pitch = tf.squeeze(pitch, axis=-1)
@@ -33,6 +42,9 @@ def predict_next_note(notes, keras_model, temperature=1.0):
33
  step = tf.maximum(0, step)
34
  duration = tf.maximum(0, duration)
35
 
 
 
 
36
  return int(pitch.numpy()), float(step.numpy()), float(duration.numpy())
37
 
38
 
@@ -88,6 +100,7 @@ def generate_music(input_text, instrument_name="Acoustic Grand Piano"):
88
  generated_notes = []
89
  for _ in range(num_predictions):
90
  pitch, step, duration = predict_next_note(input_data[-25:], model, temperature)
 
91
  generated_notes.append((pitch, step, duration))
92
  new_note = np.array([[pitch, step, duration]])
93
  input_data = np.vstack([input_data, new_note])
 
20
  assert temperature > 0
21
  inputs = tf.expand_dims(notes, 0)
22
  predictions = model.predict(inputs)
23
+
24
+
25
  pitch_logits = predictions['pitch']
26
  step = predictions['step']
27
  duration = predictions['duration']
28
 
29
+
30
+ # print("predictions: ", predictions)
31
+ # print("Shape of pitch logits:", predictions['pitch'].shape)
32
+ # print("Content of pitch logits:", predictions['pitch'])
33
+ # print("step:", step)
34
+ # print("duration:", duration)
35
+
36
  pitch_logits /= temperature
37
  pitch = tf.random.categorical(pitch_logits, num_samples=1)
38
  pitch = tf.squeeze(pitch, axis=-1)
 
42
  step = tf.maximum(0, step)
43
  duration = tf.maximum(0, duration)
44
 
45
+ print('pitch: ', pitch)
46
+ print('int(pitch.numpy()): ', int(pitch.numpy()))
47
+
48
  return int(pitch.numpy()), float(step.numpy()), float(duration.numpy())
49
 
50
 
 
100
  generated_notes = []
101
  for _ in range(num_predictions):
102
  pitch, step, duration = predict_next_note(input_data[-25:], model, temperature)
103
+
104
  generated_notes.append((pitch, step, duration))
105
  new_note = np.array([[pitch, step, duration]])
106
  input_data = np.vstack([input_data, new_note])