Upload mel.py
Browse files- mel/mel.py +77 -0
mel/mel.py
ADDED
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import warnings
|
2 |
+
from typing import Callable, Union
|
3 |
+
|
4 |
+
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
5 |
+
from diffusers.schedulers.scheduling_utils import SchedulerMixin
|
6 |
+
|
7 |
+
warnings.filterwarnings("ignore")
|
8 |
+
|
9 |
+
import numpy as np
|
10 |
+
import librosa
|
11 |
+
from PIL import Image
|
12 |
+
|
13 |
+
class Mel(ConfigMixin, SchedulerMixin):
|
14 |
+
config_name = "mel_config.json"
|
15 |
+
|
16 |
+
@register_to_config
|
17 |
+
def __init__(
|
18 |
+
self,
|
19 |
+
x_res: int = 256,
|
20 |
+
y_res: int = 256,
|
21 |
+
sample_rate: int = 22050,
|
22 |
+
n_fft: int = 2048,
|
23 |
+
hop_length: int = 512,
|
24 |
+
top_db: float = 80.0,
|
25 |
+
n_iter: int = 32,
|
26 |
+
):
|
27 |
+
self.hop_length = hop_length
|
28 |
+
self.sr = sample_rate
|
29 |
+
self.n_fft = n_fft
|
30 |
+
self.top_db = top_db
|
31 |
+
self.audio = None
|
32 |
+
self.n_iter = n_iter
|
33 |
+
self.set_resolution(x_res, y_res)
|
34 |
+
|
35 |
+
def set_resolution(self, x_res: int, y_res: int):
|
36 |
+
self.x_res = x_res
|
37 |
+
self.y_res = y_res
|
38 |
+
self.n_mels = self.y_res
|
39 |
+
self.slice_size = self.x_res * self.hop_length - 1
|
40 |
+
|
41 |
+
def load_audio(self, audio_file: str = None, raw_audio: np.ndarray = None):
|
42 |
+
if audio_file is not None:
|
43 |
+
self.audio, _ = librosa.load(audio_file, mono=True, sr=self.sr)
|
44 |
+
else:
|
45 |
+
self.audio = raw_audio
|
46 |
+
|
47 |
+
if len(self.audio) < self.x_res * self.hop_length:
|
48 |
+
self.audio = np.concatenate([self.audio, np.zeros((self.x_res * self.hop_length - len(self.audio),))])
|
49 |
+
|
50 |
+
def get_number_of_slices(self) -> int:
|
51 |
+
return len(self.audio) // self.slice_size
|
52 |
+
|
53 |
+
def get_audio_slice(self, slice: int = 0) -> int:
|
54 |
+
return self.audio[self.slice_size * slice : self.slice_size * (slice + 1)]
|
55 |
+
|
56 |
+
def get_sample_rate(self) -> int:
|
57 |
+
return self.sr
|
58 |
+
|
59 |
+
def audio_slice_to_image(self, slice: int, ref: Union[float, Callable] = np.max) -> Image.Image:
|
60 |
+
S = librosa.feature.melspectrogram(
|
61 |
+
y=self.get_audio_slice(slice),
|
62 |
+
sr=self.sr,
|
63 |
+
n_fft=self.n_fft,
|
64 |
+
hop_length=self.hop_length,
|
65 |
+
n_mels=self.n_mels,
|
66 |
+
)
|
67 |
+
log_S = librosa.power_to_db(S, ref=ref, top_db=self.top_db)
|
68 |
+
spec_data = (((log_S + self.top_db) * 255 / self.top_db).clip(0, 255) + 0.5).astype(np.uint8)
|
69 |
+
return Image.fromarray(spec_data)
|
70 |
+
|
71 |
+
def image_to_audio(self, image: Image.Image) -> np.ndarray:
|
72 |
+
spec_data = np.frombuffer(image.tobytes(), dtype=np.uint8).reshape((image.height, image.width))
|
73 |
+
log_S = spec_data.astype("float") * self.top_db / 255 - self.top_db
|
74 |
+
S = librosa.db_to_power(log_S)
|
75 |
+
return librosa.feature.inverse.mel_to_audio(
|
76 |
+
S, sr=self.sr, n_fft=self.n_fft, hop_length=self.hop_length, n_iter=self.n_iter
|
77 |
+
)
|