upl base

.gitattributes

@@ -1,35 +1,35 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+*.pt
+__pycache__/*
+tmp/*
+flagged/*
+test.py

app.py

@@ -0,0 +1,190 @@
+import os
+import torch
+import shutil
+import librosa
+import numpy as np
+import gradio as gr
+import librosa.display
+import matplotlib.pyplot as plt
+import torchvision.transforms as transforms
+from collections import Counter
+from model import EvalNet
+from PIL import Image
+from utils import *
+import warnings
+warnings.filterwarnings("ignore")
+
+classes = ['m_bel', 'f_bel', 'm_folk', 'f_folk']
+
+
+def most_common_element(input_list):
+    # 使用 Counter 统计每个元素的出现次数
+    counter = Counter(input_list)
+    # 使用 most_common 方法获取出现次数最多的元素
+    most_common_element, _ = counter.most_common(1)[0]
+    return most_common_element
+
+
+def wav_to_mel(audio_path: str, width=1.6, topdb=40):
+    create_dir('./tmp')
+    try:
+        y, sr = librosa.load(audio_path, sr=48000)
+        non_silents = librosa.effects.split(y, top_db=topdb)
+        non_silent = np.concatenate(
+            [y[start:end] for start, end in non_silents]
+        )
+        mel_spec = librosa.feature.melspectrogram(y=non_silent, sr=sr)
+        log_mel_spec = librosa.power_to_db(mel_spec, ref=np.max)
+        dur = librosa.get_duration(y=non_silent, sr=sr)
+        total_frames = log_mel_spec.shape[1]
+        step = int(width * total_frames / dur)
+        count = int(total_frames / step)
+        begin = int(0.5 * (total_frames - count * step))
+        end = begin + step * count
+        for i in range(begin, end, step):
+            librosa.display.specshow(log_mel_spec[:, i:i + step])
+            plt.axis('off')
+            plt.savefig(
+                f'./tmp/mel_{round(dur, 2)}_{i}.jpg',
+                bbox_inches='tight',
+                pad_inches=0.0
+            )
+            plt.close()
+
+    except Exception as e:
+        print(f'Error converting {audio_path} : {e}')
+
+
+def wav_to_cqt(audio_path: str, width=1.6, topdb=40):
+    create_dir('./tmp')
+    try:
+        y, sr = librosa.load(audio_path, sr=48000)
+        non_silents = librosa.effects.split(y, top_db=topdb)
+        non_silent = np.concatenate(
+            [y[start:end] for start, end in non_silents]
+        )
+        cqt_spec = librosa.cqt(y=non_silent, sr=sr)
+        log_cqt_spec = librosa.power_to_db(np.abs(cqt_spec)**2, ref=np.max)
+        dur = librosa.get_duration(y=non_silent, sr=sr)
+        total_frames = log_cqt_spec.shape[1]
+        step = int(width * total_frames / dur)
+        count = int(total_frames / step)
+        begin = int(0.5 * (total_frames - count * step))
+        end = begin + step * count
+        for i in range(begin, end, step):
+            librosa.display.specshow(log_cqt_spec[:, i:i + step])
+            plt.axis('off')
+            plt.savefig(
+                f'./tmp/cqt_{round(dur, 2)}_{i}.jpg',
+                bbox_inches='tight',
+                pad_inches=0.0
+            )
+            plt.close()
+
+    except Exception as e:
+        print(f'Error converting {audio_path} : {e}')
+
+
+def wav_to_chroma(audio_path: str, width=1.6, topdb=40):
+    create_dir('./tmp')
+    try:
+        y, sr = librosa.load(audio_path, sr=48000)
+        non_silents = librosa.effects.split(y, top_db=topdb)
+        non_silent = np.concatenate(
+            [y[start:end] for start, end in non_silents]
+        )
+        chroma_spec = librosa.feature.chroma_stft(y=non_silent, sr=sr)
+        log_chroma_spec = librosa.power_to_db(
+            np.abs(chroma_spec)**2,
+            ref=np.max
+        )
+        dur = librosa.get_duration(y=non_silent, sr=sr)
+        total_frames = log_chroma_spec.shape[1]
+        step = int(width * total_frames / dur)
+        count = int(total_frames / step)
+        begin = int(0.5 * (total_frames - count * step))
+        end = begin + step * count
+        for i in range(begin, end, step):
+            librosa.display.specshow(log_chroma_spec[:, i:i + step])
+            plt.axis('off')
+            plt.savefig(
+                f'./tmp/chroma_{round(dur, 2)}_{i}.jpg',
+                bbox_inches='tight',
+                pad_inches=0.0
+            )
+            plt.close()
+
+    except Exception as e:
+        print(f'Error converting {audio_path} : {e}')
+
+
+def embed_img(img_path, input_size=224):
+    transform = transforms.Compose([
+        transforms.Resize([input_size, input_size]),
+        transforms.ToTensor(),
+        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+    ])
+    img = Image.open(img_path).convert("RGB")
+    return transform(img).unsqueeze(0)
+
+
+def inference(wav_path, log_name, folder_path='./tmp'):
+    if os.path.exists(folder_path):
+        shutil.rmtree(folder_path)
+
+    if not wav_path:
+        wav_path = './examples/f_bel.wav'
+
+    model = EvalNet(log_name).model
+    spec = log_name.split('_')[-3]
+    eval('wav_to_%s' % spec)(wav_path)
+    outputs = []
+    all_files = os.listdir(folder_path)
+    for file_name in all_files:
+        if file_name.lower().endswith('.jpg'):
+            file_path = os.path.join(folder_path, file_name)
+            input = embed_img(file_path)
+            output = model(input)
+            pred_id = torch.max(output.data, 1)[1]
+            outputs.append(pred_id)
+
+    max_count_item = most_common_element(outputs)
+    shutil.rmtree(folder_path)
+    return translate[classes[max_count_item]]
+
+
+models = [
+    'vit_b_16_mel_2024-01-07_05-16-24',
+    'swin_b_chroma_2024-01-07_14-01-10'
+]
+
+translate = {
+    'm_bel': 'male bel canto',
+    'm_folk': 'male folk singing',
+    'f_bel': 'female bel canto',
+    'f_folk': 'female folk singing'
+}
+
+examples = []
+example_wavs = find_wav_files()
+for wav in example_wavs:
+    examples.append([
+        wav,
+        models[0]
+    ])
+
+iface = gr.Interface(
+    fn=inference,
+    inputs=[
+        gr.Audio(label='Upload audio', type='filepath'),
+        gr.Dropdown(
+            choices=models,
+            label='Select model',
+            value=models[0]
+        )
+    ],
+    outputs=gr.Textbox(label='Singing method'),
+    examples=examples
+)
+
+iface.launch()

examples/f_bel.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26abdaf26e98f1ac58a510462740ca47a569b4060917e2f413cd4a84aa0d8b66
+size 839708

examples/f_folk.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:752c041e9c44762a90b5f0983cda805bcdc09d308d564574d6146c2bfdca2d97
+size 1183688

examples/m_bel.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7b1aa8cfc6e004df1d1a7649927c06187535ce8531f3dda2177709b9d11b70d
+size 2881538

examples/m_folk.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51c3b595ae7c0a361a6364df282439aa923a1098c9b62abfa13b6e82558a10c5
+size 1154582

model.py

@@ -0,0 +1,148 @@
+import os
+import torch
+import torch.nn as nn
+import torchvision.models as models
+from modelscope.msdatasets import MsDataset
+from utils import url_download, create_dir, DOMAIN
+
+
+def get_backbone(ver, backbone_list):
+    for bb in backbone_list:
+        if ver == bb['ver']:
+            return bb
+
+    print('Backbone name not found, using default option - alexnet.')
+    return backbone_list[0]
+
+
+def model_info(m_ver):
+    backbone_list = MsDataset.load(
+        'monetjoe/cv_backbones',
+        subset_name='ImageNet1k_v1',
+        split='train'
+    )
+    backbone = get_backbone(m_ver, backbone_list)
+    m_type = str(backbone['type'])
+    input_size = int(backbone['input_size'])
+    return m_type, input_size
+
+
+def download_model(log_name='vit_b_16_mel_2024-01-07_05-16-24'):
+    pre_model_url = f'{DOMAIN}{log_name}/save.pt'
+    pre_model_path = f"./model/{log_name}.pt"
+    m_ver = '_'.join(log_name.split('_')[:-3])
+    create_dir('./model')
+
+    if not os.path.exists(pre_model_path):
+        url_download(pre_model_url, pre_model_path)
+
+    return pre_model_path, m_ver
+
+
+def Classifier(cls_num: int, output_size: int, linear_output: bool):
+    q = (1.0 * output_size / cls_num) ** 0.25
+    l1 = int(q * cls_num)
+    l2 = int(q * l1)
+    l3 = int(q * l2)
+
+    if linear_output:
+        return torch.nn.Sequential(
+            nn.Dropout(),
+            nn.Linear(output_size, l3),
+            nn.ReLU(inplace=True),
+            nn.Dropout(),
+            nn.Linear(l3, l2),
+            nn.ReLU(inplace=True),
+            nn.Dropout(),
+            nn.Linear(l2, l1),
+            nn.ReLU(inplace=True),
+            nn.Linear(l1, cls_num)
+        )
+
+    else:
+        return torch.nn.Sequential(
+            nn.Dropout(),
+            nn.Conv2d(output_size, l3, kernel_size=(1, 1), stride=(1, 1)),
+            nn.ReLU(inplace=True),
+            nn.AdaptiveAvgPool2d(output_size=(1, 1)),
+            nn.Flatten(),
+            nn.Linear(l3, l2),
+            nn.ReLU(inplace=True),
+            nn.Dropout(),
+            nn.Linear(l2, l1),
+            nn.ReLU(inplace=True),
+            nn.Linear(l1, cls_num)
+        )
+
+
+class EvalNet():
+    model = None
+    m_type = 'squeezenet'
+    input_size = 224
+    output_size = 512
+
+    def __init__(self, log_name, cls_num=4):
+        saved_model_path, m_ver = download_model(log_name)
+        self.m_type, self.input_size = model_info(m_ver)
+
+        if not hasattr(models, m_ver):
+            print('Unsupported model.')
+            exit()
+
+        self.model = eval('models.%s()' % m_ver)
+        linear_output = self._set_outsize()
+        self._set_classifier(cls_num, linear_output)
+        checkpoint = torch.load(saved_model_path, map_location='cpu')
+        if torch.cuda.is_available():
+            checkpoint = torch.load(saved_model_path)
+
+        self.model.load_state_dict(checkpoint, False)
+        self.model.eval()
+
+    def _set_outsize(self, debug_mode=False):
+        for name, module in self.model.named_modules():
+            if str(name).__contains__('classifier') or str(name).__eq__('fc') or str(name).__contains__('head'):
+                if isinstance(module, torch.nn.Linear):
+                    self.output_size = module.in_features
+                    if debug_mode:
+                        print(
+                            f"{name}(Linear): {self.output_size} -> {module.out_features}")
+                    return True
+
+                if isinstance(module, torch.nn.Conv2d):
+                    self.output_size = module.in_channels
+                    if debug_mode:
+                        print(
+                            f"{name}(Conv2d): {self.output_size} -> {module.out_channels}")
+                    return False
+
+        return False
+
+    def _set_classifier(self, cls_num, linear_output):
+        if hasattr(self.model, 'classifier'):
+            self.model.classifier = Classifier(
+                cls_num, self.output_size, linear_output)
+            return
+
+        elif hasattr(self.model, 'fc'):
+            self.model.fc = Classifier(
+                cls_num, self.output_size, linear_output)
+            return
+
+        elif hasattr(self.model, 'head'):
+            self.model.head = Classifier(
+                cls_num, self.output_size, linear_output)
+            return
+
+        self.model.heads.head = Classifier(
+            cls_num, self.output_size, linear_output)
+
+    def forward(self, x):
+        if torch.cuda.is_available():
+            x = x.cuda()
+            self.model = self.model.cuda()
+
+        if self.m_type == 'googlenet' and self.training:
+            return self.model(x)[0]
+        else:
+            return self.model(x)

requirements.txt

@@ -0,0 +1,6 @@
+librosa
+torch
+matplotlib
+torchvision
+pillow
+gradio

utils.py

@@ -0,0 +1,96 @@
+import os
+import time
+import torch
+import zipfile
+import requests
+from tqdm import tqdm
+
+DOMAIN = 'https://huggingface.co/ccmusic-database/bel_canto/resolve/main/'
+
+
+def create_dir(dir):
+    if not os.path.exists(dir):
+        os.mkdir(dir)
+
+
+def url_download(url: str, fname: str, max_retries=3):
+    retry_count = 0
+    while retry_count < max_retries:
+        try:
+            print(f"Downloading: {url}")
+            resp = requests.get(url, stream=True)
+            # Check the response status code (raise an exception if it's not in the range 200-299)
+            resp.raise_for_status()
+            total = int(resp.headers.get('content-length', 0))
+            # create_dir(data_dir)
+            with open(fname, 'wb') as file, tqdm(
+                    desc=fname,
+                    total=total,
+                    unit='iB',
+                    unit_scale=True,
+                    unit_divisor=1024,
+            ) as bar:
+                for data in resp.iter_content(chunk_size=1024):
+                    size = file.write(data)
+                    bar.update(size)
+            print(f'Download of {url} completed.')
+            return
+
+        except requests.exceptions.HTTPError as errh:
+            print(f"HTTP error occurred: {errh}")
+            retry_count += 1
+            continue
+        except requests.exceptions.ConnectionError as errc:
+            print(f"Connection error occurred: {errc}")
+            retry_count += 1
+            continue
+        except requests.exceptions.Timeout as errt:
+            print(f"Timeout error occurred: {errt}")
+            retry_count += 1
+            continue
+        except Exception as err:
+            print(f"Other error occurred: {err}")
+            retry_count += 1
+            continue
+
+    else:
+        print(
+            "Error: the operation could not be completed after {max_retries} retries."
+        )
+        exit()
+
+
+def unzip_file(zip_src, dst_dir):
+    r = zipfile.is_zipfile(zip_src)
+    if r:
+        fz = zipfile.ZipFile(zip_src, 'r')
+        for file in fz.namelist():
+            fz.extract(file, dst_dir)
+    else:
+        print('This is not zip')
+
+
+def time_stamp(timestamp=None):
+    if timestamp != None:
+        return timestamp.strftime("%Y-%m-%d %H:%M:%S")
+
+    return time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime(time.time()))
+
+
+def toCUDA(x):
+    if hasattr(x, 'cuda'):
+        if torch.cuda.is_available():
+            return x.cuda()
+
+    return x
+
+
+def find_wav_files(folder_path='./examples'):
+    wav_files = []
+    for root, _, files in os.walk(folder_path):
+        for file in files:
+            if file.endswith(".wav"):
+                file_path = os.path.join(root, file)
+                wav_files.append(file_path)
+
+    return wav_files