update

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,18 @@
+
+.git/
+.idea/
+
+**/file_dir
+**/flagged/
+**/log/
+**/logs/
+**/__pycache__/
+
+data/
+docs/
+dotenv/
+trained_models/
+temp/
+
+#**/*.wav
+**/*.xlsx

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.8
+
+WORKDIR /code
+
+COPY . /code
+
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+RUN useradd -m -u 1000 user
+
+USER user
+
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+CMD ["python3", "main.py"]

README.md

@@ -0,0 +1,11 @@
+---
+title: VM Sound Classification
+emoji: 🐢
+colorFrom: purple
+colorTo: blue
+sdk: docker
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

examples/vm_sound_classification/conv2d_classifier.yaml

@@ -0,0 +1,45 @@
+model_name: "cnn_audio_classifier"
+
+mel_spectrogram_param:
+  sample_rate: 8000
+  n_fft: 512
+  win_length: 200
+  hop_length: 80
+  f_min: 10
+  f_max: 3800
+  window_fn: hamming
+  n_mels: 80
+
+conv2d_block_param_list:
+  - batch_norm: true
+    in_channels: 1
+    out_channels: 16
+    kernel_size: 3
+    stride: 1
+    dilation: 3
+    activation: relu
+    dropout: 0.1
+  - in_channels: 16
+    out_channels: 16
+    kernel_size: 5
+    stride: 2
+    dilation: 3
+    activation: relu
+    dropout: 0.1
+  - in_channels: 16
+    out_channels: 16
+    kernel_size: 3
+    stride: 1
+    dilation: 2
+    activation: relu
+    dropout: 0.1
+
+cls_head_param:
+  input_dim: 432
+  num_layers: 2
+  hidden_dims:
+    - 128
+    - 32
+  activations: relu
+  dropout: 0.1
+  num_labels: 8

examples/vm_sound_classification/requirements.txt

@@ -0,0 +1,10 @@
+torch==1.13.1
+torchaudio==0.13.1
+fsspec==2022.1.0
+librosa==0.9.2
+pandas==1.1.5
+openpyxl==3.0.9
+xlrd==1.2.0
+tqdm==4.64.1
+overrides==1.9.0
+pyyaml==6.0.1

examples/vm_sound_classification/run.sh

@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+
+: <<'END'
+
+sh run.sh --stage 3 --stop_stage 3 --system_version windows --file_folder_name file_dir --final_model_name vm_sound_classification3 \
+--filename_patterns "E:/programmer/asr_datasets/voicemail/wav_finished/en-US/wav_finished/*/*.wav \
+E:/programmer/asr_datasets/voicemail/wav_finished/id-ID/wav_finished/*/*.wav" \
+
+sh run.sh --stage 0 --stop_stage 1 --system_version windows --file_folder_name file_dir --final_model_name vm_sound_classification3 \
+--filename_patterns "E:/programmer/asr_datasets/voicemail/wav_finished/en-US/wav_finished/*/*.wav \
+E:/programmer/asr_datasets/voicemail/wav_finished/id-ID/wav_finished/*/*.wav" \
+
+sh run.sh --stage 2 --stop_stage 2 --system_version windows --file_folder_name file_dir --final_model_name vm_sound_classification3
+sh run.sh --stage 3 --stop_stage 3 --system_version windows --file_folder_name file_dir --final_model_name vm_sound_classification3
+
+sh run.sh --stage 3 --stop_stage 5 --system_version centos --file_folder_name file_dir --final_model_name vm_sound_classification8-ch16 \
+--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav"
+
+
+"
+
+END
+
+
+# params
+system_version="windows";
+verbose=true;
+stage=0 # start from 0 if you need to start from data preparation
+stop_stage=9
+
+work_dir="$(pwd)"
+file_folder_name=file_folder_name
+final_model_name=final_model_name
+filename_patterns="/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav"
+nohup_name=nohup.out
+
+country=en-US
+
+# model params
+batch_size=64
+max_epochs=200
+save_top_k=10
+patience=5
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+file_dir="${work_dir}/${file_folder_name}"
+final_model_dir="${work_dir}/../../trained_models/${final_model_name}";
+
+dataset="${file_dir}/dataset.xlsx"
+train_dataset="${file_dir}/train.xlsx"
+valid_dataset="${file_dir}/valid.xlsx"
+evaluation_file="${file_dir}/evaluation.xlsx"
+vocabulary_dir="${file_dir}/vocabulary"
+
+$verbose && echo "system_version: ${system_version}"
+$verbose && echo "file_folder_name: ${file_folder_name}"
+
+if [ $system_version == "windows" ]; then
+  alias python3='D:/Users/tianx/PycharmProjects/virtualenv/vm_sound_classification/Scripts/python.exe'
+elif [ $system_version == "centos" ] || [ $system_version == "ubuntu" ]; then
+  #source /data/local/bin/vm_sound_classification/bin/activate
+  alias python3='/data/local/bin/vm_sound_classification/bin/python3'
+fi
+
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+  $verbose && echo "stage 0: prepare data"
+  cd "${work_dir}" || exit 1
+  python3 step_1_prepare_data.py \
+  --file_dir "${file_dir}" \
+  --filename_patterns "${filename_patterns}" \
+  --train_dataset "${train_dataset}" \
+  --valid_dataset "${valid_dataset}" \
+
+fi
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  $verbose && echo "stage 1: make vocabulary"
+  cd "${work_dir}" || exit 1
+  python3 step_2_make_vocabulary.py \
+  --vocabulary_dir "${vocabulary_dir}" \
+  --train_dataset "${train_dataset}" \
+  --valid_dataset "${valid_dataset}" \
+
+fi
+
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  $verbose && echo "stage 2: train model"
+  cd "${work_dir}" || exit 1
+  python3 step_3_train_model.py \
+  --vocabulary_dir "${vocabulary_dir}" \
+  --train_dataset "${train_dataset}" \
+  --valid_dataset "${valid_dataset}" \
+  --serialization_dir "${file_dir}" \
+
+fi
+
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  $verbose && echo "stage 3: test model"
+  cd "${work_dir}" || exit 1
+  python3 step_4_evaluation_model.py \
+  --dataset "${dataset}" \
+  --vocabulary_dir "${vocabulary_dir}" \
+  --model_dir "${file_dir}/best" \
+  --output_file "${evaluation_file}" \
+
+fi
+
+
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+  $verbose && echo "stage 4: export model"
+  cd "${work_dir}" || exit 1
+  python3 step_5_export_models.py \
+  --vocabulary_dir "${vocabulary_dir}" \
+  --model_dir "${file_dir}/best" \
+  --serialization_dir "${file_dir}" \
+
+fi
+
+
+if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
+  $verbose && echo "stage 5: collect files"
+  cd "${work_dir}" || exit 1
+
+  mkdir -p ${final_model_dir}
+
+  cp "${file_dir}/best"/* "${final_model_dir}"
+  cp -r "${file_dir}/vocabulary" "${final_model_dir}"
+
+  cp "${file_dir}/evaluation.xlsx" "${final_model_dir}/evaluation.xlsx"
+
+  cp "${file_dir}/trace_model.zip" "${final_model_dir}/trace_model.zip"
+  cp "${file_dir}/trace_quant_model.zip" "${final_model_dir}/trace_quant_model.zip"
+  cp "${file_dir}/script_model.zip" "${final_model_dir}/script_model.zip"
+  cp "${file_dir}/script_quant_model.zip" "${final_model_dir}/script_quant_model.zip"
+
+  cd "${final_model_dir}/.." || exit 1;
+
+  if [ -e "${final_model_name}.zip" ]; then
+    rm -rf "${final_model_name}_backup.zip"
+    mv "${final_model_name}.zip" "${final_model_name}_backup.zip"
+  fi
+
+  zip -r "${final_model_name}.zip" "${final_model_name}"
+  rm -rf "${final_model_name}"
+
+fi
+
+
+if [ ${stage} -le 6 ] && [ ${stop_stage} -ge 6 ]; then
+  $verbose && echo "stage 6: clear file_dir"
+  cd "${work_dir}" || exit 1
+
+  rm -rf "${file_dir}";
+
+fi

examples/vm_sound_classification/step_1_prepare_data.py

@@ -0,0 +1,150 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from glob import glob
+import json
+import os
+from pathlib import Path
+import random
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import pandas as pd
+from scipy.io import wavfile
+from tqdm import tqdm
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--file_dir", default="./", type=str)
+    parser.add_argument("--task", default="default", type=str)
+    parser.add_argument("--filename_patterns", type=str)
+
+    parser.add_argument("--train_dataset", default="train.xlsx", type=str)
+    parser.add_argument("--valid_dataset", default="valid.xlsx", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def get_dataset(args):
+    filename_patterns = args.filename_patterns
+    filename_patterns = filename_patterns.split(" ")
+    print(filename_patterns)
+
+    file_dir = Path(args.file_dir)
+    file_dir.mkdir(exist_ok=True)
+
+    # label3_map = {
+    #     "bell": "voicemail",
+    #     "white_noise": "mute",
+    #     "low_white_noise": "mute",
+    #     "high_white_noise": "mute",
+    #     # "music": "music",
+    #     "mute": "mute",
+    #     "noise": "voice_or_noise",
+    #     "noise_mute": "voice_or_noise",
+    #     "voice": "voice_or_noise",
+    #     "voicemail": "voicemail",
+    # }
+    label8_map = {
+        "bell": "bell",
+        "white_noise": "white_noise",
+        "low_white_noise": "white_noise",
+        "high_white_noise": "white_noise",
+        "music": "music",
+        "mute": "mute",
+        "noise": "noise",
+        "noise_mute": "noise_mute",
+        "voice": "voice",
+        "voicemail": "voicemail",
+    }
+
+    result = list()
+    for filename_pattern in filename_patterns:
+        filename_list = glob(filename_pattern)
+        for filename in tqdm(filename_list):
+            filename = Path(filename)
+            sample_rate, signal = wavfile.read(filename.as_posix())
+            if len(signal) < sample_rate * 2:
+                continue
+
+            folder = filename.parts[-2]
+            country = filename.parts[-4]
+
+            if folder not in label8_map.keys():
+                continue
+
+            labels = label8_map[folder]
+
+            random1 = random.random()
+            random2 = random.random()
+
+            result.append({
+                "filename": filename,
+                "folder": folder,
+                "category": country,
+                "labels": labels,
+                "random1": random1,
+                "random2": random2,
+                "flag": "TRAIN" if random2 < 0.8 else "TEST",
+            })
+
+    df = pd.DataFrame(result)
+    pivot_table = pd.pivot_table(df, index=["labels"], values=["filename"], aggfunc="count")
+    print(pivot_table)
+
+    df = df.sort_values(by=["random1"], ascending=False)
+    df.to_excel(
+        file_dir / "dataset.xlsx",
+        index=False,
+        # encoding="utf_8_sig"
+    )
+
+    return
+
+
+def split_dataset(args):
+    """分割训练集, 测试集"""
+    file_dir = Path(args.file_dir)
+    file_dir.mkdir(exist_ok=True)
+
+    df = pd.read_excel(file_dir / "dataset.xlsx")
+
+    train = list()
+    test = list()
+
+    for i, row in df.iterrows():
+        flag = row["flag"]
+        if flag == "TRAIN":
+            train.append(row)
+        else:
+            test.append(row)
+
+    train = pd.DataFrame(train)
+    train.to_excel(
+        args.train_dataset,
+        index=False,
+        # encoding="utf_8_sig"
+    )
+    test = pd.DataFrame(test)
+    test.to_excel(
+        args.valid_dataset,
+        index=False,
+        # encoding="utf_8_sig"
+    )
+
+    return
+
+
+def main():
+    args = get_args()
+    get_dataset(args)
+    split_dataset(args)
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/vm_sound_classification/step_2_make_vocabulary.py

@@ -0,0 +1,51 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+from pathlib import Path
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import pandas as pd
+
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vocabulary_dir", default="vocabulary", type=str)
+
+    parser.add_argument("--train_dataset", default="train.xlsx", type=str)
+    parser.add_argument("--valid_dataset", default="valid.xlsx", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    train_dataset = pd.read_excel(args.train_dataset)
+    valid_dataset = pd.read_excel(args.valid_dataset)
+
+    vocabulary = Vocabulary()
+
+    # train
+    for i, row in train_dataset.iterrows():
+        label = row["labels"]
+        vocabulary.add_token_to_namespace(label, namespace="labels")
+
+    # valid
+    for i, row in valid_dataset.iterrows():
+        label = row["labels"]
+        vocabulary.add_token_to_namespace(label, namespace="labels")
+
+    vocabulary.save_to_files(args.vocabulary_dir)
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/vm_sound_classification/step_3_train_model.py

@@ -0,0 +1,331 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from collections import defaultdict
+import json
+import logging
+from logging.handlers import TimedRotatingFileHandler
+import os
+import platform
+from pathlib import Path
+import random
+import sys
+import shutil
+from typing import List
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import numpy as np
+import torch
+from torch.utils.data.dataloader import DataLoader
+from tqdm import tqdm
+
+from toolbox.torch.modules.loss import FocalLoss, HingeLoss, HingeLinear
+from toolbox.torch.training.metrics.categorical_accuracy import CategoricalAccuracy
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+from toolbox.torch.utils.data.dataset.wave_classifier_excel_dataset import WaveClassifierExcelDataset
+from toolbox.torchaudio.models.cnn_audio_classifier.modeling_cnn_audio_classifier import WaveClassifierPretrainedModel
+from toolbox.torchaudio.models.cnn_audio_classifier.configuration_cnn_audio_classifier import CnnAudioClassifierConfig
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vocabulary_dir", default="vocabulary", type=str)
+
+    parser.add_argument("--train_dataset", default="train.xlsx", type=str)
+    parser.add_argument("--valid_dataset", default="valid.xlsx", type=str)
+
+    parser.add_argument("--max_epochs", default=100, type=int)
+
+    parser.add_argument("--batch_size", default=64, type=int)
+    parser.add_argument("--learning_rate", default=1e-3, type=float)
+    parser.add_argument("--num_serialized_models_to_keep", default=10, type=int)
+    parser.add_argument("--patience", default=5, type=int)
+    parser.add_argument("--serialization_dir", default="serialization_dir", type=str)
+    parser.add_argument("--seed", default=0, type=int)
+
+    parser.add_argument("--config_file", default="conv2d_classifier.yaml", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def logging_config(file_dir: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    logging.basicConfig(format=fmt,
+                        datefmt="%m/%d/%Y %H:%M:%S",
+                        level=logging.DEBUG)
+    file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(file_dir, "main.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(logging.Formatter(fmt))
+    logger = logging.getLogger(__name__)
+    logger.addHandler(file_handler)
+
+    return logger
+
+
+class CollateFunction(object):
+    def __init__(self):
+        pass
+
+    def __call__(self, batch: List[dict]):
+        array_list = list()
+        label_list = list()
+        for sample in batch:
+            array = sample["waveform"]
+            label = sample["label"]
+
+            array_list.append(array)
+            label_list.append(label)
+
+        array_list = torch.stack(array_list)
+        label_list = torch.stack(label_list)
+        return array_list, label_list
+
+
+collate_fn = CollateFunction()
+
+
+def main():
+    args = get_args()
+
+    serialization_dir = Path(args.serialization_dir)
+    serialization_dir.mkdir(parents=True, exist_ok=True)
+
+    logger = logging_config(serialization_dir)
+
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    logger.info("set seed: {}".format(args.seed))
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    n_gpu = torch.cuda.device_count()
+    logger.info("GPU available count: {}; device: {}".format(n_gpu, device))
+
+    vocabulary = Vocabulary.from_files(args.vocabulary_dir)
+
+    # datasets
+    logger.info("prepare datasets")
+    train_dataset = WaveClassifierExcelDataset(
+        vocab=vocabulary,
+        excel_file=args.train_dataset,
+        category=None,
+        category_field="category",
+        label_field="labels",
+        expected_sample_rate=8000,
+        max_wave_value=32768.0,
+    )
+    valid_dataset = WaveClassifierExcelDataset(
+        vocab=vocabulary,
+        excel_file=args.valid_dataset,
+        category=None,
+        category_field="category",
+        label_field="labels",
+        expected_sample_rate=8000,
+        max_wave_value=32768.0,
+    )
+    train_data_loader = DataLoader(
+        dataset=train_dataset,
+        batch_size=args.batch_size,
+        shuffle=True,
+        # Linux 系统中可以使用多个子进程加载数据, 而在 Windows 系统中不能.
+        num_workers=0 if platform.system() == "Windows" else os.cpu_count() // 2,
+        collate_fn=collate_fn,
+        pin_memory=False,
+        # prefetch_factor=64,
+    )
+    valid_data_loader = DataLoader(
+        dataset=valid_dataset,
+        batch_size=args.batch_size,
+        shuffle=True,
+        # Linux 系统中可以使用多个子进程加载数据, 而在 Windows 系统中不能.
+        num_workers=0 if platform.system() == "Windows" else os.cpu_count() // 2,
+        collate_fn=collate_fn,
+        pin_memory=False,
+        # prefetch_factor=64,
+    )
+
+    # models
+    logger.info("prepare models")
+    config = CnnAudioClassifierConfig.from_pretrained(
+        pretrained_model_name_or_path=args.config_file,
+        # num_labels=vocabulary.get_vocab_size(namespace="labels")
+    )
+    if not config.cls_head_param["num_labels"] == vocabulary.get_vocab_size(namespace="labels"):
+        raise AssertionError
+    model = WaveClassifierPretrainedModel(
+        config=config,
+    )
+    model.to(device)
+    model.train()
+
+    # optimizer
+    logger.info("prepare optimizer, lr_scheduler, loss_fn, categorical_accuracy")
+    param_optimizer = model.parameters()
+    optimizer = torch.optim.Adam(
+        param_optimizer,
+        lr=args.learning_rate,
+    )
+    # lr_scheduler = torch.optim.lr_scheduler.StepLR(
+    #     optimizer,
+    #     step_size=2000
+    # )
+    lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
+        optimizer,
+        milestones=[10000, 20000, 30000], gamma=0.5
+    )
+    focal_loss = FocalLoss(
+        num_classes=vocabulary.get_vocab_size(namespace="labels"),
+        reduction="mean",
+    )
+    categorical_accuracy = CategoricalAccuracy()
+
+    # training loop
+    logger.info("training")
+
+    training_loss = 10000000000
+    training_accuracy = 0.
+    evaluation_loss = 10000000000
+    evaluation_accuracy = 0.
+
+    model_list = list()
+    best_idx_epoch = None
+    best_accuracy = None
+    patience_count = 0
+
+    for idx_epoch in range(args.max_epochs):
+        categorical_accuracy.reset()
+        total_loss = 0.
+        total_examples = 0.
+        progress_bar = tqdm(
+            total=len(train_data_loader),
+            desc="Training; epoch: {}".format(idx_epoch),
+        )
+        for batch in train_data_loader:
+            input_ids, label_ids = batch
+            input_ids = input_ids.to(device)
+            label_ids: torch.LongTensor = label_ids.to(device).long()
+
+            logits = model.forward(input_ids)
+            loss = focal_loss.forward(logits, label_ids.view(-1))
+            categorical_accuracy(logits, label_ids)
+
+            total_loss += loss.item()
+            total_examples += input_ids.size(0)
+
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            lr_scheduler.step()
+
+            training_loss = total_loss / total_examples
+            training_loss = round(training_loss, 4)
+            training_accuracy = categorical_accuracy.get_metric()["accuracy"]
+            training_accuracy = round(training_accuracy, 4)
+
+            progress_bar.update(1)
+            progress_bar.set_postfix({
+                "training_loss": training_loss,
+                "training_accuracy": training_accuracy,
+            })
+
+        categorical_accuracy.reset()
+        total_loss = 0.
+        total_examples = 0.
+        progress_bar = tqdm(
+            total=len(valid_data_loader),
+            desc="Evaluation; epoch: {}".format(idx_epoch),
+        )
+        for batch in valid_data_loader:
+            input_ids, label_ids = batch
+            input_ids = input_ids.to(device)
+            label_ids: torch.LongTensor = label_ids.to(device).long()
+
+            with torch.no_grad():
+                logits = model.forward(input_ids)
+                loss = focal_loss.forward(logits, label_ids.view(-1))
+                categorical_accuracy(logits, label_ids)
+
+            total_loss += loss.item()
+            total_examples += input_ids.size(0)
+
+            evaluation_loss = total_loss / total_examples
+            evaluation_loss = round(evaluation_loss, 4)
+            evaluation_accuracy = categorical_accuracy.get_metric()["accuracy"]
+            evaluation_accuracy = round(evaluation_accuracy, 4)
+
+            progress_bar.update(1)
+            progress_bar.set_postfix({
+                "evaluation_loss": evaluation_loss,
+                "evaluation_accuracy": evaluation_accuracy,
+            })
+
+        # save path
+        epoch_dir = serialization_dir / "epoch-{}".format(idx_epoch)
+        epoch_dir.mkdir(parents=True, exist_ok=False)
+
+        # save models
+        model.save_pretrained(epoch_dir.as_posix())
+
+        model_list.append(epoch_dir)
+        if len(model_list) >= args.num_serialized_models_to_keep:
+            model_to_delete: Path = model_list.pop(0)
+            shutil.rmtree(model_to_delete.as_posix())
+
+        # save metric
+        if best_accuracy is None:
+            best_idx_epoch = idx_epoch
+            best_accuracy = evaluation_accuracy
+        elif evaluation_accuracy > best_accuracy:
+            best_idx_epoch = idx_epoch
+            best_accuracy = evaluation_accuracy
+        else:
+            pass
+
+        metrics = {
+            "idx_epoch": idx_epoch,
+            "best_idx_epoch": best_idx_epoch,
+            "best_accuracy": best_accuracy,
+            "training_loss": training_loss,
+            "training_accuracy": training_accuracy,
+            "evaluation_loss": evaluation_loss,
+            "evaluation_accuracy": evaluation_accuracy,
+            "learning_rate": optimizer.param_groups[0]['lr'],
+        }
+        metrics_filename = epoch_dir / "metrics_epoch.json"
+        with open(metrics_filename, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=4, ensure_ascii=False)
+
+        # save best
+        best_dir = serialization_dir / "best"
+        if best_idx_epoch == idx_epoch:
+            if best_dir.exists():
+                shutil.rmtree(best_dir)
+            shutil.copytree(epoch_dir, best_dir)
+
+        # early stop
+        early_stop_flag = False
+        if best_idx_epoch == idx_epoch:
+            patience_count = 0
+        else:
+            patience_count += 1
+        if patience_count >= args.patience:
+            early_stop_flag = True
+
+        # early stop
+        if early_stop_flag:
+            break
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/vm_sound_classification/step_4_evaluation_model.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from collections import defaultdict
+import json
+import logging
+from logging.handlers import TimedRotatingFileHandler
+import os
+import platform
+from pathlib import Path
+import sys
+import shutil
+from typing import List
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import pandas as pd
+from scipy.io import wavfile
+import torch
+from tqdm import tqdm
+
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+from toolbox.torchaudio.models.cnn_audio_classifier.modeling_cnn_audio_classifier import WaveClassifierPretrainedModel
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset", default="dataset.xlsx", type=str)
+
+    parser.add_argument("--vocabulary_dir", default="vocabulary", type=str)
+    parser.add_argument("--model_dir", default="best", type=str)
+
+    parser.add_argument("--output_file", default="evaluation.xlsx", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def logging_config():
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    logging.basicConfig(format=fmt,
+                        datefmt="%m/%d/%Y %H:%M:%S",
+                        level=logging.DEBUG)
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.INFO)
+    stream_handler.setFormatter(logging.Formatter(fmt))
+
+    logger = logging.getLogger(__name__)
+
+    return logger
+
+
+def main():
+    args = get_args()
+
+    logger = logging_config()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    n_gpu = torch.cuda.device_count()
+    logger.info("GPU available count: {}; device: {}".format(n_gpu, device))
+
+    logger.info("prepare vocabulary, model")
+    vocabulary = Vocabulary.from_files(args.vocabulary_dir)
+
+    model = WaveClassifierPretrainedModel.from_pretrained(
+        pretrained_model_name_or_path=args.model_dir,
+    )
+    model.to(device)
+    model.eval()
+
+    logger.info("read excel")
+    df = pd.read_excel(args.dataset)
+    result = list()
+
+    total_correct = 0
+    total_examples = 0
+
+    progress_bar = tqdm(total=len(df), desc="Evaluation")
+    for i, row in df.iterrows():
+        filename = row["filename"]
+        ground_true = row["labels"]
+
+        sample_rate, waveform = wavfile.read(filename)
+        waveform = waveform / (1 << 15)
+        waveform = torch.tensor(waveform, dtype=torch.float32)
+        waveform = torch.unsqueeze(waveform, dim=0)
+        waveform = waveform.to(device)
+
+        with torch.no_grad():
+            logits = model.forward(waveform)
+            probs = torch.nn.functional.softmax(logits, dim=-1)
+            label_idx = torch.argmax(probs, dim=-1)
+
+        label_idx = label_idx.cpu()
+        probs = probs.cpu()
+
+        label_idx = label_idx.numpy()[0]
+        label_str = vocabulary.get_token_from_index(label_idx, namespace="labels")
+        prob = probs[0][label_idx].numpy()
+
+        correct = 1 if label_str == ground_true else 0
+        row_ = dict(row)
+        row_["predict"] = label_str
+        row_["prob"] = prob
+        row_["correct"] = correct
+        result.append(row_)
+
+        total_examples += 1
+        total_correct += correct
+        accuracy = total_correct / total_examples
+
+        progress_bar.update(1)
+        progress_bar.set_postfix({
+            "accuracy": accuracy,
+        })
+
+    result = pd.DataFrame(result)
+    result.to_excel(
+        args.output_file,
+        index=False
+    )
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/vm_sound_classification/step_5_export_models.py

@@ -0,0 +1,106 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from collections import defaultdict
+import json
+import logging
+from logging.handlers import TimedRotatingFileHandler
+import os
+import platform
+from pathlib import Path
+import sys
+import shutil
+from typing import List
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import numpy as np
+import torch
+
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+from toolbox.torchaudio.models.cnn_audio_classifier.modeling_cnn_audio_classifier import WaveClassifierPretrainedModel
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vocabulary_dir", default="vocabulary", type=str)
+    parser.add_argument("--model_dir", default="best", type=str)
+
+    parser.add_argument("--serialization_dir", default="serialization_dir", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def logging_config():
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    logging.basicConfig(format=fmt,
+                        datefmt="%m/%d/%Y %H:%M:%S",
+                        level=logging.DEBUG)
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.INFO)
+    stream_handler.setFormatter(logging.Formatter(fmt))
+
+    logger = logging.getLogger(__name__)
+
+    return logger
+
+
+def main():
+    args = get_args()
+
+    serialization_dir = Path(args.serialization_dir)
+
+    logger = logging_config()
+
+    logger.info("export models on CPU")
+    device = torch.device("cpu")
+
+    logger.info("prepare vocabulary, model")
+    vocabulary = Vocabulary.from_files(args.vocabulary_dir)
+
+    model = WaveClassifierPretrainedModel.from_pretrained(
+        pretrained_model_name_or_path=args.model_dir,
+        num_labels=vocabulary.get_vocab_size(namespace="labels")
+    )
+    model.to(device)
+    model.eval()
+
+    waveform = 0 + 25 * np.random.randn(16000,)
+    waveform = np.array(waveform, dtype=np.int16)
+    waveform = waveform / (1 << 15)
+    waveform = torch.tensor(waveform, dtype=torch.float32)
+    waveform = torch.unsqueeze(waveform, dim=0)
+    waveform = waveform.to(device)
+
+    logger.info("export jit models")
+    example_inputs = (waveform,)
+
+    # trace model
+    trace_model = torch.jit.trace(func=model, example_inputs=example_inputs, strict=False)
+    trace_model.save(serialization_dir / "trace_model.zip")
+
+    # quantization trace model (not work on GPU)
+    quantized_model = torch.quantization.quantize_dynamic(
+        model, {torch.nn.Linear}, dtype=torch.qint8
+    )
+    trace_quant_model = torch.jit.trace(func=quantized_model, example_inputs=example_inputs, strict=False)
+    trace_quant_model.save(serialization_dir / "trace_quant_model.zip")
+
+    # script model
+    script_model = torch.jit.script(obj=model)
+    script_model.save(serialization_dir / "script_model.zip")
+
+    # quantization script model (not work on GPU)
+    quantized_model = torch.quantization.quantize_dynamic(
+        model, {torch.nn.Linear}, dtype=torch.qint8
+    )
+    script_quant_model = torch.jit.script(quantized_model)
+    script_quant_model.save(serialization_dir / "script_quant_model.zip")
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/vm_sound_classification/step_6_infer.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+import zipfile
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+from scipy.io import wavfile
+import torch
+
+from project_settings import project_path
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_file",
+        default=(project_path / "trained_models/vm_sound_classification3.zip").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--wav_file",
+        default=r"C:\Users\tianx\Desktop\a073d03d-d280-46df-9b2d-d904965f4500_zh-CN_h3f25ivhb0c0_1719478037746.wav",
+        type=str
+    )
+
+    parser.add_argument("--device", default="cpu", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    model_file = Path(args.model_file)
+
+    device = torch.device(args.device)
+
+    with zipfile.ZipFile(model_file, "r") as f_zip:
+        out_root = Path(tempfile.gettempdir()) / "vm_sound_classification"
+        print(out_root.as_posix())
+        if out_root.exists():
+            shutil.rmtree(out_root.as_posix())
+        out_root.mkdir(parents=True, exist_ok=True)
+        f_zip.extractall(path=out_root)
+
+    tgt_path = out_root / model_file.stem
+    jit_model_file = tgt_path / "trace_model.zip"
+    vocab_path = tgt_path / "vocabulary"
+
+    with open(jit_model_file.as_posix(), "rb") as f:
+        model = torch.jit.load(f)
+    model.to(device)
+    model.eval()
+    vocabulary = Vocabulary.from_files(vocab_path.as_posix())
+
+    # infer
+    sample_rate, waveform = wavfile.read(args.wav_file)
+    waveform = waveform[:16000]
+    waveform = waveform / (1 << 15)
+    waveform = torch.tensor(waveform, dtype=torch.float32)
+    waveform = torch.unsqueeze(waveform, dim=0)
+    waveform = waveform.to(device)
+
+    with torch.no_grad():
+        logits = model.forward(waveform)
+        probs = torch.nn.functional.softmax(logits, dim=-1)
+        label_idx = torch.argmax(probs, dim=-1)
+
+    label_idx = label_idx.cpu()
+    probs = probs.cpu()
+
+    label_idx = label_idx.numpy()[0]
+    prob = probs.numpy()[0][label_idx]
+
+    label_str = vocabulary.get_token_from_index(label_idx, namespace="labels")
+    print(label_str)
+    print(prob)
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/vm_sound_classification/step_7_test_model.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+import zipfile
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+from scipy.io import wavfile
+import torch
+
+from project_settings import project_path
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+from toolbox.torchaudio.models.cnn_audio_classifier.modeling_cnn_audio_classifier import WaveClassifierPretrainedModel
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_file",
+        default=(project_path / "trained_models/vm_sound_classification3.zip").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--wav_file",
+        default=r"C:\Users\tianx\Desktop\4b284733-0be3-4a48-abbb-615b32ac44b7_6ndddc2szlh0.wav",
+        type=str
+    )
+
+    parser.add_argument("--device", default="cpu", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    model_file = Path(args.model_file)
+
+    device = torch.device(args.device)
+
+    with zipfile.ZipFile(model_file, "r") as f_zip:
+        out_root = Path(tempfile.gettempdir()) / "vm_sound_classification"
+        print(out_root)
+        if out_root.exists():
+            shutil.rmtree(out_root.as_posix())
+        out_root.mkdir(parents=True, exist_ok=True)
+        f_zip.extractall(path=out_root)
+
+    tgt_path = out_root / model_file.stem
+    vocab_path = tgt_path / "vocabulary"
+
+    vocabulary = Vocabulary.from_files(vocab_path.as_posix())
+
+    model = WaveClassifierPretrainedModel.from_pretrained(
+        pretrained_model_name_or_path=tgt_path.as_posix(),
+    )
+    model.to(device)
+    model.eval()
+
+    # infer
+    sample_rate, waveform = wavfile.read(args.wav_file)
+    waveform = waveform[:16000]
+    waveform = waveform / (1 << 15)
+    waveform = torch.tensor(waveform, dtype=torch.float32)
+    waveform = torch.unsqueeze(waveform, dim=0)
+    waveform = waveform.to(device)
+    print(waveform.shape)
+    with torch.no_grad():
+        logits = model.forward(waveform)
+        probs = torch.nn.functional.softmax(logits, dim=-1)
+        label_idx = torch.argmax(probs, dim=-1)
+
+    label_idx = label_idx.cpu()
+    probs = probs.cpu()
+
+    label_idx = label_idx.numpy()[0]
+    prob = probs.numpy()[0][label_idx]
+
+    label_str = vocabulary.get_token_from_index(label_idx, namespace="labels")
+    print(label_str)
+    print(prob)
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/vm_sound_classification/stop.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+kill -9 `ps -aef | grep 'vm_sound_classification/bin/python3' | grep -v grep | awk '{print $2}' | sed 's/\n/ /'`

examples/vm_sound_classification8/requirements.txt

@@ -0,0 +1,9 @@
+torch==1.10.1
+torchaudio==0.10.1
+fsspec==2022.1.0
+librosa==0.9.2
+pandas==1.1.5
+openpyxl==3.0.9
+xlrd==1.2.0
+tqdm==4.64.1
+overrides==1.9.0

examples/vm_sound_classification8/run.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+
+: <<'END'
+
+sh run.sh --stage 3 --stop_stage 3 --system_version windows --file_folder_name file_dir --final_model_name vm_sound_classification8 \
+--filename_patterns "E:/programmer/asr_datasets/voicemail/wav_finished/en-US/wav_finished/*/*.wav \
+E:/programmer/asr_datasets/voicemail/wav_finished/id-ID/wav_finished/*/*.wav" \
+
+
+sh run.sh --stage 3 --stop_stage 3 --system_version windows --file_folder_name file_dir --final_model_name vm_sound_classification8 \
+--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav"
+
+sh run.sh --stage 4 --stop_stage 4 --system_version windows --file_folder_name file_dir --final_model_name vm_sound_classification8 \
+--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav"
+
+sh run.sh --stage 4 --stop_stage 4 --system_version centos --file_folder_name file_dir --final_model_name vm_sound_classification8 \
+--filename_patterns "/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav"
+
+
+"
+
+END
+
+
+# sh run.sh --stage -1 --stop_stage 9
+# sh run.sh --stage -1 --stop_stage 5 --system_version centos --file_folder_name task_cnn_voicemail_id_id --final_model_name cnn_voicemail_id_id
+# sh run.sh --stage 3 --stop_stage 4
+# sh run.sh --stage 4 --stop_stage 4
+# sh run.sh --stage 3 --stop_stage 3 --system_version centos --file_folder_name task_cnn_voicemail_id_id
+
+# params
+system_version="windows";
+verbose=true;
+stage=0 # start from 0 if you need to start from data preparation
+stop_stage=9
+
+work_dir="$(pwd)"
+file_folder_name=file_folder_name
+final_model_name=final_model_name
+filename_patterns="/data/tianxing/PycharmProjects/datasets/voicemail/*/wav_finished/*/*.wav"
+nohup_name=nohup.out
+
+country=en-US
+
+# model params
+batch_size=64
+max_epochs=200
+save_top_k=10
+patience=5
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+file_dir="${work_dir}/${file_folder_name}"
+final_model_dir="${work_dir}/../../trained_models/${final_model_name}";
+
+train_dataset="${file_dir}/train.xlsx"
+valid_dataset="${file_dir}/valid.xlsx"
+vocabulary_dir="${file_dir}/vocabulary"
+
+
+$verbose && echo "system_version: ${system_version}"
+$verbose && echo "file_folder_name: ${file_folder_name}"
+
+if [ $system_version == "windows" ]; then
+  alias python3='D:/Users/tianx/PycharmProjects/virtualenv/vm_sound_classification/Scripts/python.exe'
+elif [ $system_version == "centos" ] || [ $system_version == "ubuntu" ]; then
+  #source /data/local/bin/vm_sound_classification/bin/activate
+  alias python3='/data/local/bin/vm_sound_classification/bin/python3'
+fi
+
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+  $verbose && echo "stage 0: prepare data"
+  cd "${work_dir}" || exit 1
+  python3 step_1_prepare_data.py \
+  --file_dir "${file_dir}" \
+  --filename_patterns "${filename_patterns}" \
+  --train_dataset "${train_dataset}" \
+  --valid_dataset "${valid_dataset}" \
+
+fi
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  $verbose && echo "stage 1: make vocabulary"
+  cd "${work_dir}" || exit 1
+  python3 step_2_make_vocabulary.py \
+  --vocabulary_dir "${vocabulary_dir}" \
+  --train_dataset "${train_dataset}" \
+  --valid_dataset "${valid_dataset}" \
+
+fi
+
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  $verbose && echo "stage 2: train global model"
+  cd "${work_dir}" || exit 1
+  python3 step_3_train_global_model.py \
+  --vocabulary_dir "${vocabulary_dir}" \
+  --train_dataset "${train_dataset}" \
+  --valid_dataset "${valid_dataset}" \
+  --serialization_dir "${file_dir}/global_model" \
+
+fi
+
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  $verbose && echo "stage 3: train country model"
+  cd "${work_dir}" || exit 1
+  python3 step_4_train_country_model.py \
+  --vocabulary_dir "${vocabulary_dir}" \
+  --train_dataset "${train_dataset}" \
+  --valid_dataset "${valid_dataset}" \
+  --country "${country}" \
+  --serialization_dir "${file_dir}/country_model" \
+
+fi
+
+
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+  $verbose && echo "stage 4: train union model"
+  cd "${work_dir}" || exit 1
+  python3 step_5_train_union.py \
+  --vocabulary_dir "${vocabulary_dir}" \
+  --train_dataset "${train_dataset}" \
+  --valid_dataset "${valid_dataset}" \
+  --serialization_dir "${file_dir}/union" \
+
+fi

examples/vm_sound_classification8/step_1_prepare_data.py

@@ -0,0 +1,156 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from glob import glob
+import json
+import os
+from pathlib import Path
+import random
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import pandas as pd
+from scipy.io import wavfile
+from tqdm import tqdm
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--file_dir", default="./", type=str)
+    parser.add_argument("--task", default="default", type=str)
+    parser.add_argument("--filename_patterns", type=str)
+
+    parser.add_argument("--train_dataset", default="train.xlsx", type=str)
+    parser.add_argument("--valid_dataset", default="valid.xlsx", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def get_dataset(args):
+    filename_patterns = args.filename_patterns
+    filename_patterns = filename_patterns.split(" ")
+    print(filename_patterns)
+
+    file_dir = Path(args.file_dir)
+    file_dir.mkdir(exist_ok=True)
+
+    global_label_map = {
+        "bell": "bell",
+        "white_noise": "white_noise",
+        "low_white_noise": "white_noise",
+        "high_white_noise": "noise",
+        "music": "music",
+        "mute": "mute",
+        "noise": "noise",
+        "noise_mute": "noise_mute",
+        "voice": "voice",
+        "voicemail": "voicemail",
+    }
+
+    country_label_map = {
+        "bell": "voicemail",
+        "white_noise": "non_voicemail",
+        "low_white_noise": "non_voicemail",
+        "hight_white_noise": "non_voicemail",
+        "music": "non_voicemail",
+        "mute": "non_voicemail",
+        "noise": "non_voicemail",
+        "noise_mute": "non_voicemail",
+        "voice": "non_voicemail",
+        "voicemail": "voicemail",
+        "non_voicemail": "non_voicemail",
+    }
+
+    result = list()
+    for filename_pattern in filename_patterns:
+        filename_list = glob(filename_pattern)
+        for filename in tqdm(filename_list):
+            filename = Path(filename)
+            sample_rate, signal = wavfile.read(filename.as_posix())
+            if len(signal) < sample_rate * 2:
+                continue
+
+            folder = filename.parts[-2]
+            country = filename.parts[-4]
+
+            if folder not in global_label_map.keys():
+                continue
+            if folder not in country_label_map.keys():
+                continue
+
+            global_label = global_label_map[folder]
+            country_label = country_label_map[folder]
+
+            random1 = random.random()
+            random2 = random.random()
+
+            result.append({
+                "filename": filename,
+                "folder": folder,
+                "category": country,
+                "global_labels": global_label,
+                "country_labels": country_label,
+                "random1": random1,
+                "random2": random2,
+                "flag": "TRAIN" if random2 < 0.8 else "TEST",
+            })
+
+    df = pd.DataFrame(result)
+    pivot_table = pd.pivot_table(df, index=["global_labels"], values=["filename"], aggfunc="count")
+    print(pivot_table)
+
+    df = df.sort_values(by=["random1"], ascending=False)
+    df.to_excel(
+        file_dir / "dataset.xlsx",
+        index=False,
+        # encoding="utf_8_sig"
+    )
+
+    return
+
+
+def split_dataset(args):
+    """分割训练集, 测试集"""
+    file_dir = Path(args.file_dir)
+    file_dir.mkdir(exist_ok=True)
+
+    df = pd.read_excel(file_dir / "dataset.xlsx")
+
+    train = list()
+    test = list()
+
+    for i, row in df.iterrows():
+        flag = row["flag"]
+        if flag == "TRAIN":
+            train.append(row)
+        else:
+            test.append(row)
+
+    train = pd.DataFrame(train)
+    train.to_excel(
+        args.train_dataset,
+        index=False,
+        # encoding="utf_8_sig"
+    )
+    test = pd.DataFrame(test)
+    test.to_excel(
+        args.valid_dataset,
+        index=False,
+        # encoding="utf_8_sig"
+    )
+
+    return
+
+
+def main():
+    args = get_args()
+    get_dataset(args)
+    split_dataset(args)
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/vm_sound_classification8/step_2_make_vocabulary.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+from pathlib import Path
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import pandas as pd
+
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vocabulary_dir", default="vocabulary", type=str)
+
+    parser.add_argument("--train_dataset", default="train.xlsx", type=str)
+    parser.add_argument("--valid_dataset", default="valid.xlsx", type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    train_dataset = pd.read_excel(args.train_dataset)
+    valid_dataset = pd.read_excel(args.valid_dataset)
+
+    # non_padded_namespaces
+    category_set = set()
+    for i, row in train_dataset.iterrows():
+        category = row["category"]
+        category_set.add(category)
+
+    for i, row in valid_dataset.iterrows():
+        category = row["category"]
+        category_set.add(category)
+
+    vocabulary = Vocabulary(non_padded_namespaces=["global_labels", *list(category_set)])
+
+    # train
+    for i, row in train_dataset.iterrows():
+        global_labels = row["global_labels"]
+        country_labels = row["country_labels"]
+        category = row["category"]
+
+        vocabulary.add_token_to_namespace(global_labels, "global_labels")
+        vocabulary.add_token_to_namespace(country_labels, category)
+
+    # valid
+    for i, row in valid_dataset.iterrows():
+        global_labels = row["global_labels"]
+        country_labels = row["country_labels"]
+        category = row["category"]
+
+        vocabulary.add_token_to_namespace(global_labels, "global_labels")
+        vocabulary.add_token_to_namespace(country_labels, category)
+
+    vocabulary.save_to_files(args.vocabulary_dir)
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/vm_sound_classification8/step_3_train_global_model.py

@@ -0,0 +1,328 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+之前的代码达到准确率0.8423
+此代码达到准确率0.8379
+此代码可行.
+"""
+import argparse
+import copy
+import json
+import logging
+from logging.handlers import TimedRotatingFileHandler
+import os
+from pathlib import Path
+import platform
+import sys
+from typing import List
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import torch
+from torch.utils.data.dataloader import DataLoader
+from tqdm import tqdm
+
+from toolbox.torch.modules.loss import FocalLoss, HingeLoss, HingeLinear
+from toolbox.torch.training.metrics.categorical_accuracy import CategoricalAccuracy
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+from toolbox.torch.utils.data.dataset.wave_classifier_excel_dataset import WaveClassifierExcelDataset
+from toolbox.torchaudio.models.cnn_audio_classifier.modeling_cnn_audio_classifier import WaveEncoder, ClsHead, WaveClassifier
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vocabulary_dir", default="vocabulary", type=str)
+
+    parser.add_argument("--train_dataset", default="train.xlsx", type=str)
+    parser.add_argument("--valid_dataset", default="valid.xlsx", type=str)
+
+    parser.add_argument("--max_epochs", default=100, type=int)
+    parser.add_argument("--batch_size", default=64, type=int)
+    parser.add_argument("--learning_rate", default=1e-3, type=float)
+    parser.add_argument("--num_serialized_models_to_keep", default=10, type=int)
+    parser.add_argument("--patience", default=5, type=int)
+    parser.add_argument("--serialization_dir", default="global_classifier", type=str)
+    parser.add_argument("--seed", default=0, type=int)
+
+    args = parser.parse_args()
+    return args
+
+
+def logging_config(file_dir: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    logging.basicConfig(format=fmt,
+                        datefmt="%m/%d/%Y %H:%M:%S",
+                        level=logging.DEBUG)
+    file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(file_dir, "main.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(logging.Formatter(fmt))
+    logger = logging.getLogger(__name__)
+    logger.addHandler(file_handler)
+
+    return logger
+
+
+class CollateFunction(object):
+    def __init__(self):
+        pass
+
+    def __call__(self, batch: List[dict]):
+        array_list = list()
+        label_list = list()
+        for sample in batch:
+            array = sample["waveform"]
+            label = sample["label"]
+
+            array_list.append(array)
+            label_list.append(label)
+
+        array_list = torch.stack(array_list)
+        label_list = torch.stack(label_list)
+        return array_list, label_list
+
+
+collate_fn = CollateFunction()
+
+
+def main():
+    args = get_args()
+
+    serialization_dir = Path(args.serialization_dir)
+    serialization_dir.mkdir(parents=True, exist_ok=True)
+
+    logger = logging_config(args.serialization_dir)
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    n_gpu = torch.cuda.device_count()
+    logger.info("GPU available: {}; device: {}".format(n_gpu, device))
+
+    vocabulary = Vocabulary.from_files(args.vocabulary_dir)
+
+    # datasets
+    train_dataset = WaveClassifierExcelDataset(
+        vocab=vocabulary,
+        excel_file=args.train_dataset,
+        category=None,
+        category_field="category",
+        label_field="global_labels",
+        expected_sample_rate=8000,
+        max_wave_value=32768.0,
+    )
+    valid_dataset = WaveClassifierExcelDataset(
+        vocab=vocabulary,
+        excel_file=args.valid_dataset,
+        category=None,
+        category_field="category",
+        label_field="global_labels",
+        expected_sample_rate=8000,
+        max_wave_value=32768.0,
+    )
+
+    train_data_loader = DataLoader(
+        dataset=train_dataset,
+        batch_size=args.batch_size,
+        shuffle=True,
+        # Linux 系统中可以使用多个子进程加载数据, 而在 Windows 系统中不能.
+        num_workers=0 if platform.system() == "Windows" else os.cpu_count(),
+        collate_fn=collate_fn,
+        pin_memory=False,
+        # prefetch_factor=64,
+    )
+    valid_data_loader = DataLoader(
+        dataset=valid_dataset,
+        batch_size=args.batch_size,
+        shuffle=True,
+        # Linux 系统中可以使用多个子进程加载数据, 而在 Windows 系统中不能.
+        num_workers=0 if platform.system() == "Windows" else os.cpu_count(),
+        collate_fn=collate_fn,
+        pin_memory=False,
+        # prefetch_factor=64,
+    )
+
+    # models - classifier
+    wave_encoder = WaveEncoder(
+        conv1d_block_param_list=[
+            {
+                'batch_norm': True,
+                'in_channels': 80,
+                'out_channels': 16,
+                'kernel_size': 3,
+                'stride': 3,
+                # 'padding': 'same',
+                'activation': 'relu',
+                'dropout': 0.1,
+            },
+            {
+                # 'batch_norm': True,
+                'in_channels': 16,
+                'out_channels': 16,
+                'kernel_size': 3,
+                'stride': 3,
+                # 'padding': 'same',
+                'activation': 'relu',
+                'dropout': 0.1,
+            },
+            {
+                # 'batch_norm': True,
+                'in_channels': 16,
+                'out_channels': 16,
+                'kernel_size': 3,
+                'stride': 3,
+                # 'padding': 'same',
+                'activation': 'relu',
+                'dropout': 0.1,
+            },
+        ],
+        mel_spectrogram_param={
+            "sample_rate": 8000,
+            "n_fft": 512,
+            "win_length": 200,
+            "hop_length": 80,
+            "f_min": 10,
+            "f_max": 3800,
+            "window_fn": "hamming",
+            "n_mels": 80,
+        }
+    )
+    cls_head = ClsHead(
+        input_dim=16,
+        num_layers=2,
+        hidden_dims=[32, 16],
+        activations="relu",
+        dropout=0.1,
+        num_labels=vocabulary.get_vocab_size(namespace="global_labels")
+    )
+    model = WaveClassifier(
+        wave_encoder=wave_encoder,
+        cls_head=cls_head,
+    )
+    model.to(device)
+
+    # optimizer
+    optimizer = torch.optim.Adam(
+        model.parameters(),
+        lr=args.learning_rate
+    )
+    lr_scheduler = torch.optim.lr_scheduler.StepLR(
+        optimizer,
+        step_size=30000
+    )
+    focal_loss = FocalLoss(
+        num_classes=vocabulary.get_vocab_size(namespace="global_labels"),
+        reduction="mean",
+    )
+    categorical_accuracy = CategoricalAccuracy()
+
+    # training
+    best_idx_epoch: int = None
+    best_accuracy: float = None
+    patience_count = 0
+    global_step = 0
+    model_filename_list = list()
+    for idx_epoch in range(args.max_epochs):
+
+        # training
+        model.train()
+        total_loss = 0
+        total_examples = 0
+        for step, batch in enumerate(tqdm(train_data_loader, desc="Epoch={} (training)".format(idx_epoch))):
+            input_ids, label_ids = batch
+            input_ids = input_ids.to(device)
+            label_ids: torch.LongTensor = label_ids.to(device).long()
+
+            logits = model.forward(input_ids)
+            loss = focal_loss.forward(logits, label_ids.view(-1))
+            categorical_accuracy(logits, label_ids)
+
+            total_loss += loss.item()
+            total_examples += input_ids.size(0)
+
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            lr_scheduler.step()
+
+            global_step += 1
+        training_loss = total_loss / total_examples
+        training_loss = round(training_loss, 4)
+        training_accuracy = categorical_accuracy.get_metric(reset=True)["accuracy"]
+        training_accuracy = round(training_accuracy, 4)
+        logger.info("Epoch: {}; training_loss: {}; training_accuracy: {}".format(
+            idx_epoch, training_loss, training_accuracy
+        ))
+
+        # evaluation
+        model.eval()
+        total_loss = 0
+        total_examples = 0
+        for step, batch in enumerate(tqdm(valid_data_loader, desc="Epoch={} (evaluation)".format(idx_epoch))):
+            input_ids, label_ids = batch
+            input_ids = input_ids.to(device)
+            label_ids: torch.LongTensor = label_ids.to(device).long()
+
+            with torch.no_grad():
+                logits = model.forward(input_ids)
+                loss = focal_loss.forward(logits, label_ids.view(-1))
+                categorical_accuracy(logits, label_ids)
+
+            total_loss += loss.item()
+            total_examples += input_ids.size(0)
+
+        evaluation_loss = total_loss / total_examples
+        evaluation_loss = round(evaluation_loss, 4)
+        evaluation_accuracy = categorical_accuracy.get_metric(reset=True)["accuracy"]
+        evaluation_accuracy = round(evaluation_accuracy, 4)
+        logger.info("Epoch: {}; evaluation_loss: {}; evaluation_accuracy: {}".format(
+            idx_epoch, evaluation_loss, evaluation_accuracy
+        ))
+
+        # save metric
+        metrics = {
+            "training_loss": training_loss,
+            "training_accuracy": training_accuracy,
+            "evaluation_loss": evaluation_loss,
+            "evaluation_accuracy": evaluation_accuracy,
+            "best_idx_epoch": best_idx_epoch,
+            "best_accuracy": best_accuracy,
+        }
+        metrics_filename = os.path.join(args.serialization_dir, "metrics_epoch_{}.json".format(idx_epoch))
+        with open(metrics_filename, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=4, ensure_ascii=False)
+
+        # save model
+        model_filename = os.path.join(args.serialization_dir, "model_epoch_{}.bin".format(idx_epoch))
+        model_filename_list.append(model_filename)
+        if len(model_filename_list) >= args.num_serialized_models_to_keep:
+            model_filename_to_delete = model_filename_list.pop(0)
+            os.remove(model_filename_to_delete)
+        torch.save(model.state_dict(), model_filename)
+
+        # early stop
+        best_model_filename = os.path.join(args.serialization_dir, "best.bin")
+        if best_accuracy is None:
+            best_idx_epoch = idx_epoch
+            best_accuracy = evaluation_accuracy
+            torch.save(model.state_dict(), best_model_filename)
+        elif evaluation_accuracy > best_accuracy:
+            best_idx_epoch = idx_epoch
+            best_accuracy = evaluation_accuracy
+            torch.save(model.state_dict(), best_model_filename)
+            patience_count = 0
+        elif patience_count >= args.patience:
+            break
+        else:
+            patience_count += 1
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/vm_sound_classification8/step_4_train_country_model.py

@@ -0,0 +1,349 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+只训练 cls_head 部分的参数, 模型的准确率会更低.
+"""
+import argparse
+from collections import defaultdict
+import json
+import logging
+from logging.handlers import TimedRotatingFileHandler
+import os
+import platform
+from pathlib import Path
+import sys
+import shutil
+from typing import List
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import pandas as pd
+import torch
+from torch.utils.data.dataloader import DataLoader
+from tqdm import tqdm
+
+from toolbox.torch.modules.loss import FocalLoss, HingeLoss, HingeLinear
+from toolbox.torch.training.metrics.categorical_accuracy import CategoricalAccuracy
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+from toolbox.torch.utils.data.dataset.wave_classifier_excel_dataset import WaveClassifierExcelDataset
+from toolbox.torchaudio.models.cnn_audio_classifier.modeling_cnn_audio_classifier import WaveEncoder, ClsHead, WaveClassifier
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vocabulary_dir", default="vocabulary", type=str)
+    parser.add_argument("--train_dataset", default="train.xlsx", type=str)
+    parser.add_argument("--valid_dataset", default="valid.xlsx", type=str)
+
+    parser.add_argument("--country", default="en-US", type=str)
+    parser.add_argument("--shared_encoder", default="file_dir/global_model/best.bin", type=str)
+
+    parser.add_argument("--max_epochs", default=100, type=int)
+    parser.add_argument("--batch_size", default=64, type=int)
+    parser.add_argument("--learning_rate", default=1e-3, type=float)
+    parser.add_argument("--num_serialized_models_to_keep", default=10, type=int)
+    parser.add_argument("--patience", default=5, type=int)
+    parser.add_argument("--serialization_dir", default="country_models", type=str)
+    parser.add_argument("--seed", default=0, type=int)
+
+    args = parser.parse_args()
+    return args
+
+
+def logging_config(file_dir: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    logging.basicConfig(format=fmt,
+                        datefmt="%m/%d/%Y %H:%M:%S",
+                        level=logging.DEBUG)
+    file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(file_dir, "main.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(logging.Formatter(fmt))
+    logger = logging.getLogger(__name__)
+    logger.addHandler(file_handler)
+
+    return logger
+
+
+class CollateFunction(object):
+    def __init__(self):
+        pass
+
+    def __call__(self, batch: List[dict]):
+        array_list = list()
+        label_list = list()
+        for sample in batch:
+            array = sample['waveform']
+            label = sample['label']
+
+            array_list.append(array)
+            label_list.append(label)
+
+        array_list = torch.stack(array_list)
+        label_list = torch.stack(label_list)
+        return array_list, label_list
+
+
+collate_fn = CollateFunction()
+
+
+def main():
+    args = get_args()
+
+    serialization_dir = Path(args.serialization_dir)
+    serialization_dir.mkdir(parents=True, exist_ok=True)
+
+    logger = logging_config(args.serialization_dir)
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    n_gpu = torch.cuda.device_count()
+    logger.info("GPU available: {}; device: {}".format(n_gpu, device))
+
+    vocabulary = Vocabulary.from_files(args.vocabulary_dir)
+
+    # datasets
+    logger.info("prepare datasets")
+    train_dataset = WaveClassifierExcelDataset(
+        vocab=vocabulary,
+        excel_file=args.train_dataset,
+        category=args.country,
+        category_field="category",
+        label_field="country_labels",
+        expected_sample_rate=8000,
+        max_wave_value=32768.0,
+    )
+    valid_dataset = WaveClassifierExcelDataset(
+        vocab=vocabulary,
+        excel_file=args.valid_dataset,
+        category=args.country,
+        category_field="category",
+        label_field="country_labels",
+        expected_sample_rate=8000,
+        max_wave_value=32768.0,
+    )
+
+    train_data_loader = DataLoader(
+        dataset=train_dataset,
+        batch_size=args.batch_size,
+        shuffle=True,
+        # Linux 系统中可以使用多个子进程加载数据, 而在 Windows 系统中不能.
+        num_workers=0 if platform.system() == "Windows" else os.cpu_count(),
+        collate_fn=collate_fn,
+        pin_memory=False,
+        # prefetch_factor=64,
+    )
+    valid_data_loader = DataLoader(
+        dataset=valid_dataset,
+        batch_size=args.batch_size,
+        shuffle=True,
+        # Linux 系统中可以使用多个子进程加载数据, 而在 Windows 系统中不能.
+        num_workers=0 if platform.system() == "Windows" else os.cpu_count(),
+        collate_fn=collate_fn,
+        pin_memory=False,
+        # prefetch_factor=64,
+    )
+
+    # models - classifier
+    wave_encoder = WaveEncoder(
+        conv1d_block_param_list=[
+            {
+                'batch_norm': True,
+                'in_channels': 80,
+                'out_channels': 16,
+                'kernel_size': 3,
+                'stride': 3,
+                # 'padding': 'same',
+                'activation': 'relu',
+                'dropout': 0.1,
+            },
+            {
+                # 'batch_norm': True,
+                'in_channels': 16,
+                'out_channels': 16,
+                'kernel_size': 3,
+                'stride': 3,
+                # 'padding': 'same',
+                'activation': 'relu',
+                'dropout': 0.1,
+            },
+            {
+                # 'batch_norm': True,
+                'in_channels': 16,
+                'out_channels': 16,
+                'kernel_size': 3,
+                'stride': 3,
+                # 'padding': 'same',
+                'activation': 'relu',
+                'dropout': 0.1,
+            },
+        ],
+        mel_spectrogram_param={
+            "sample_rate": 8000,
+            "n_fft": 512,
+            "win_length": 200,
+            "hop_length": 80,
+            "f_min": 10,
+            "f_max": 3800,
+            "window_fn": "hamming",
+            "n_mels": 80,
+        }
+    )
+
+    with open(args.shared_encoder, "rb") as f:
+        state_dict = torch.load(f, map_location=device)
+    processed_state_dict = dict()
+    prefix = "wave_encoder."
+    for k, v in state_dict.items():
+        if not str(k).startswith(prefix):
+            continue
+        k = k[len(prefix):]
+        processed_state_dict[k] = v
+
+    wave_encoder.load_state_dict(
+        state_dict=processed_state_dict,
+        strict=True,
+    )
+    cls_head = ClsHead(
+        input_dim=16,
+        num_layers=2,
+        hidden_dims=[32, 16],
+        activations="relu",
+        dropout=0.1,
+        num_labels=vocabulary.get_vocab_size(namespace="global_labels")
+    )
+    model = WaveClassifier(
+        wave_encoder=wave_encoder,
+        cls_head=cls_head,
+    )
+    model.wave_encoder.requires_grad_(requires_grad=False)
+    model.cls_head.requires_grad_(requires_grad=True)
+    model.to(device)
+
+    # optimizer
+    logger.info("prepare optimizer")
+    optimizer = torch.optim.Adam(
+        model.cls_head.parameters(),
+        lr=args.learning_rate,
+    )
+    lr_scheduler = torch.optim.lr_scheduler.StepLR(
+        optimizer,
+        step_size=2000
+    )
+    focal_loss = FocalLoss(
+        num_classes=vocabulary.get_vocab_size(namespace=args.country),
+        reduction="mean",
+    )
+    categorical_accuracy = CategoricalAccuracy()
+
+    # training loop
+    best_idx_epoch: int = None
+    best_accuracy: float = None
+    patience_count = 0
+    global_step = 0
+    model_filename_list = list()
+    for idx_epoch in range(args.max_epochs):
+
+        # training
+        model.train()
+        total_loss = 0
+        total_examples = 0
+        for step, batch in enumerate(tqdm(train_data_loader, desc="Epoch={} (training)".format(idx_epoch))):
+            input_ids, label_ids = batch
+            input_ids = input_ids.to(device)
+            label_ids: torch.LongTensor = label_ids.to(device).long()
+
+            logits = model.forward(input_ids)
+            loss = focal_loss.forward(logits, label_ids.view(-1))
+            categorical_accuracy(logits, label_ids)
+
+            total_loss += loss.item()
+            total_examples += input_ids.size(0)
+
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            lr_scheduler.step()
+
+            global_step += 1
+        training_loss = total_loss / total_examples
+        training_loss = round(training_loss, 4)
+        training_accuracy = categorical_accuracy.get_metric(reset=True)["accuracy"]
+        training_accuracy = round(training_accuracy, 4)
+        logger.info("Epoch: {}; training_loss: {}; training_accuracy: {}".format(
+            idx_epoch, training_loss, training_accuracy
+        ))
+
+        # evaluation
+        model.eval()
+        total_loss = 0
+        total_examples = 0
+        for step, batch in enumerate(tqdm(valid_data_loader, desc="Epoch={} (evaluation)".format(idx_epoch))):
+            input_ids, label_ids = batch
+            input_ids = input_ids.to(device)
+            label_ids: torch.LongTensor = label_ids.to(device).long()
+
+            with torch.no_grad():
+                logits = model.forward(input_ids)
+                loss = focal_loss.forward(logits, label_ids.view(-1))
+                categorical_accuracy(logits, label_ids)
+
+            total_loss += loss.item()
+            total_examples += input_ids.size(0)
+
+        evaluation_loss = total_loss / total_examples
+        evaluation_loss = round(evaluation_loss, 4)
+        evaluation_accuracy = categorical_accuracy.get_metric(reset=True)["accuracy"]
+        evaluation_accuracy = round(evaluation_accuracy, 4)
+        logger.info("Epoch: {}; evaluation_loss: {}; evaluation_accuracy: {}".format(
+            idx_epoch, evaluation_loss, evaluation_accuracy
+        ))
+
+        # save metric
+        metrics = {
+            "training_loss": training_loss,
+            "training_accuracy": training_accuracy,
+            "evaluation_loss": evaluation_loss,
+            "evaluation_accuracy": evaluation_accuracy,
+            "best_idx_epoch": best_idx_epoch,
+            "best_accuracy": best_accuracy,
+        }
+        metrics_filename = os.path.join(args.serialization_dir, "metrics_epoch_{}.json".format(idx_epoch))
+        with open(metrics_filename, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=4, ensure_ascii=False)
+
+        # save model
+        model_filename = os.path.join(args.serialization_dir, "model_epoch_{}.bin".format(idx_epoch))
+        model_filename_list.append(model_filename)
+        if len(model_filename_list) >= args.num_serialized_models_to_keep:
+            model_filename_to_delete = model_filename_list.pop(0)
+            os.remove(model_filename_to_delete)
+        torch.save(model.state_dict(), model_filename)
+
+        # early stop
+        best_model_filename = os.path.join(args.serialization_dir, "best.bin")
+        if best_accuracy is None:
+            best_idx_epoch = idx_epoch
+            best_accuracy = evaluation_accuracy
+            torch.save(model.state_dict(), best_model_filename)
+        elif evaluation_accuracy > best_accuracy:
+            best_idx_epoch = idx_epoch
+            best_accuracy = evaluation_accuracy
+            torch.save(model.state_dict(), best_model_filename)
+            patience_count = 0
+        elif patience_count >= args.patience:
+            break
+        else:
+            patience_count += 1
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/vm_sound_classification8/step_5_train_union.py

@@ -0,0 +1,499 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from collections import defaultdict
+import json
+import logging
+from logging.handlers import TimedRotatingFileHandler
+import os
+import platform
+from pathlib import Path
+import sys
+import shutil
+from typing import List
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+import pandas as pd
+import torch
+from torch.utils.data.dataloader import DataLoader
+from tqdm import tqdm
+
+from toolbox.torch.modules.loss import FocalLoss, HingeLoss, HingeLinear
+from toolbox.torch.training.metrics.categorical_accuracy import CategoricalAccuracy
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+from toolbox.torch.utils.data.dataset.wave_classifier_excel_dataset import WaveClassifierExcelDataset
+from toolbox.torchaudio.models.cnn_audio_classifier.modeling_cnn_audio_classifier import WaveEncoder, ClsHead, WaveClassifier
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vocabulary_dir", default="vocabulary", type=str)
+
+    parser.add_argument("--train_dataset", default="train.xlsx", type=str)
+    parser.add_argument("--valid_dataset", default="valid.xlsx", type=str)
+
+    parser.add_argument("--max_steps", default=100000, type=int)
+    parser.add_argument("--save_steps", default=30, type=int)
+
+    parser.add_argument("--batch_size", default=1, type=int)
+    parser.add_argument("--learning_rate", default=1e-3, type=float)
+    parser.add_argument("--num_serialized_models_to_keep", default=10, type=int)
+    parser.add_argument("--patience", default=5, type=int)
+    parser.add_argument("--serialization_dir", default="union", type=str)
+    parser.add_argument("--seed", default=0, type=int)
+
+    parser.add_argument("--num_workers", default=0, type=int)
+
+    args = parser.parse_args()
+    return args
+
+
+def logging_config(file_dir: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    logging.basicConfig(format=fmt,
+                        datefmt="%m/%d/%Y %H:%M:%S",
+                        level=logging.DEBUG)
+    file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(file_dir, "main.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(logging.Formatter(fmt))
+    logger = logging.getLogger(__name__)
+    logger.addHandler(file_handler)
+
+    return logger
+
+
+class CollateFunction(object):
+    def __init__(self):
+        pass
+
+    def __call__(self, batch: List[dict]):
+        array_list = list()
+        label_list = list()
+        for sample in batch:
+            array = sample['waveform']
+            label = sample['label']
+
+            array_list.append(array)
+            label_list.append(label)
+
+        array_list = torch.stack(array_list)
+        label_list = torch.stack(label_list)
+        return array_list, label_list
+
+
+collate_fn = CollateFunction()
+
+
+class DatasetIterator(object):
+    def __init__(self, data_loader: DataLoader):
+        self.data_loader = data_loader
+        self.data_loader_iter = iter(self.data_loader)
+
+    def next(self):
+        try:
+            result = self.data_loader_iter.__next__()
+        except StopIteration:
+            self.data_loader_iter = iter(self.data_loader)
+            result = self.data_loader_iter.__next__()
+        return result
+
+
+def main():
+    args = get_args()
+
+    serialization_dir = Path(args.serialization_dir)
+    serialization_dir.mkdir(parents=True, exist_ok=True)
+
+    logger = logging_config(args.serialization_dir)
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    n_gpu = torch.cuda.device_count()
+    logger.info("GPU available count: {}; device: {}".format(n_gpu, device))
+
+    vocabulary = Vocabulary.from_files(args.vocabulary_dir)
+    namespaces = vocabulary._token_to_index.keys()
+
+    # namespace_to_ratio
+    max_radio = (len(namespaces) - 1) * 3
+    namespace_to_ratio = {n: 1 for n in namespaces}
+    namespace_to_ratio["global_labels"] = max_radio
+
+    # datasets
+    logger.info("prepare datasets")
+    namespace_to_datasets = dict()
+    for namespace in namespaces:
+        logger.info("prepare datasets - {}".format(namespace))
+        if namespace == "global_labels":
+            train_dataset = WaveClassifierExcelDataset(
+                vocab=vocabulary,
+                excel_file=args.train_dataset,
+                category=None,
+                category_field="category",
+                label_field="global_labels",
+                expected_sample_rate=8000,
+                max_wave_value=32768.0,
+            )
+            valid_dataset = WaveClassifierExcelDataset(
+                vocab=vocabulary,
+                excel_file=args.valid_dataset,
+                category=None,
+                category_field="category",
+                label_field="global_labels",
+                expected_sample_rate=8000,
+                max_wave_value=32768.0,
+            )
+        else:
+            train_dataset = WaveClassifierExcelDataset(
+                vocab=vocabulary,
+                excel_file=args.train_dataset,
+                category=namespace,
+                category_field="category",
+                label_field="country_labels",
+                expected_sample_rate=8000,
+                max_wave_value=32768.0,
+            )
+            valid_dataset = WaveClassifierExcelDataset(
+                vocab=vocabulary,
+                excel_file=args.valid_dataset,
+                category=namespace,
+                category_field="category",
+                label_field="country_labels",
+                expected_sample_rate=8000,
+                max_wave_value=32768.0,
+            )
+
+        train_data_loader = DataLoader(
+            dataset=train_dataset,
+            batch_size=args.batch_size,
+            shuffle=True,
+            # Linux 系统中可以使用多个子进程加载数据, 而在 Windows 系统中不能.
+            # num_workers=0 if platform.system() == "Windows" else os.cpu_count() // 2,
+            num_workers=args.num_workers,
+            collate_fn=collate_fn,
+            pin_memory=False,
+            # prefetch_factor=64,
+        )
+        valid_data_loader = DataLoader(
+            dataset=valid_dataset,
+            batch_size=args.batch_size,
+            shuffle=True,
+            # Linux 系统中可以使用多个子进程加载数据, 而在 Windows 系统中不能.
+            # num_workers=0 if platform.system() == "Windows" else os.cpu_count() // 2,
+            num_workers=args.num_workers,
+            collate_fn=collate_fn,
+            pin_memory=False,
+            # prefetch_factor=64,
+        )
+
+        namespace_to_datasets[namespace] = {
+            "train_data_loader": train_data_loader,
+            "valid_data_loader": valid_data_loader,
+        }
+
+    # datasets iterator
+    logger.info("prepare datasets iterator")
+    namespace_to_datasets_iter = dict()
+    for namespace in namespaces:
+        logger.info("prepare datasets iterator - {}".format(namespace))
+        train_data_loader = namespace_to_datasets[namespace]["train_data_loader"]
+        valid_data_loader = namespace_to_datasets[namespace]["valid_data_loader"]
+        namespace_to_datasets_iter[namespace] = {
+            "train_data_loader_iter": DatasetIterator(train_data_loader),
+            "valid_data_loader_iter": DatasetIterator(valid_data_loader),
+        }
+
+    # models - encoder
+    logger.info("prepare models - encoder")
+    wave_encoder = WaveEncoder(
+        conv2d_block_param_list=[
+            {
+                "batch_norm": True,
+                "in_channels": 1,
+                "out_channels": 4,
+                "kernel_size": 3,
+                "stride": 1,
+                # "padding": "same",
+                "dilation": 3,
+                "activation": "relu",
+                "dropout": 0.1,
+            },
+            {
+                # "batch_norm": True,
+                "in_channels": 4,
+                "out_channels": 4,
+                "kernel_size": 5,
+                "stride": 2,
+                # "padding": "same",
+                "dilation": 3,
+                "activation": "relu",
+                "dropout": 0.1,
+            },
+            {
+                # "batch_norm": True,
+                "in_channels": 4,
+                "out_channels": 4,
+                "kernel_size": 3,
+                "stride": 1,
+                # "padding": "same",
+                "dilation": 2,
+                "activation": "relu",
+                "dropout": 0.1,
+            },
+        ],
+        mel_spectrogram_param={
+            'sample_rate': 8000,
+            'n_fft': 512,
+            'win_length': 200,
+            'hop_length': 80,
+            'f_min': 10,
+            'f_max': 3800,
+            'window_fn': 'hamming',
+            'n_mels': 80,
+        }
+    )
+
+    # models - cls_head
+    logger.info("prepare models - cls_head")
+    namespace_to_cls_heads = dict()
+    for namespace in namespaces:
+        logger.info("prepare models - cls_head - {}".format(namespace))
+        cls_head = ClsHead(
+            input_dim=352,
+            num_layers=2,
+            hidden_dims=[128, 32],
+            activations="relu",
+            dropout=0.1,
+            num_labels=vocabulary.get_vocab_size(namespace=namespace)
+        )
+        namespace_to_cls_heads[namespace] = cls_head
+
+    # models - classifier
+    logger.info("prepare models - classifier")
+    namespace_to_classifier = dict()
+    for namespace in namespaces:
+        logger.info("prepare models - classifier - {}".format(namespace))
+        cls_head = namespace_to_cls_heads[namespace]
+        wave_classifier = WaveClassifier(
+            wave_encoder=wave_encoder,
+            cls_head=cls_head,
+        )
+        wave_classifier.to(device)
+        namespace_to_classifier[namespace] = wave_classifier
+
+    # optimizer
+    logger.info("prepare optimizer")
+    param_optimizer = list()
+    param_optimizer.extend(wave_encoder.parameters())
+    for _, cls_head in namespace_to_cls_heads.items():
+        param_optimizer.extend(cls_head.parameters())
+
+    optimizer = torch.optim.Adam(
+        param_optimizer,
+        lr=args.learning_rate,
+    )
+    lr_scheduler = torch.optim.lr_scheduler.StepLR(
+        optimizer,
+        step_size=10000
+    )
+    focal_loss = FocalLoss(
+        num_classes=vocabulary.get_vocab_size(namespace="global_labels"),
+        reduction="mean",
+    )
+
+    # categorical_accuracy
+    logger.info("prepare categorical_accuracy")
+    namespace_to_categorical_accuracy = dict()
+    for namespace in namespaces:
+        categorical_accuracy = CategoricalAccuracy()
+        namespace_to_categorical_accuracy[namespace] = categorical_accuracy
+
+    # training loop
+    logger.info("prepare training loop")
+
+    model_list = list()
+    best_idx_step = None
+    best_accuracy = None
+    patience_count = 0
+
+    namespace_to_total_loss = defaultdict(float)
+    namespace_to_total_examples = defaultdict(int)
+    for idx_step in tqdm(range(args.max_steps)):
+
+        # training one step
+        loss: torch.Tensor = None
+        for namespace in namespaces:
+            train_data_loader_iter = namespace_to_datasets_iter[namespace]["train_data_loader_iter"]
+
+            ratio = namespace_to_ratio[namespace]
+            model = namespace_to_classifier[namespace]
+            categorical_accuracy = namespace_to_categorical_accuracy[namespace]
+
+            model.train()
+
+            for _ in range(ratio):
+                batch = train_data_loader_iter.next()
+                input_ids, label_ids = batch
+                input_ids = input_ids.to(device)
+                label_ids: torch.LongTensor = label_ids.to(device).long()
+
+                logits = model.forward(input_ids)
+                task_loss = focal_loss.forward(logits, label_ids.view(-1))
+                categorical_accuracy(logits, label_ids)
+
+                if loss is None:
+                    loss = task_loss
+                else:
+                    loss += task_loss
+
+                namespace_to_total_loss[namespace] += task_loss.item()
+                namespace_to_total_examples[namespace] += input_ids.size(0)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        lr_scheduler.step()
+
+        # logging
+        if (idx_step + 1) % args.save_steps == 0:
+            metrics = dict()
+
+            # training
+            for namespace in namespaces:
+                total_loss = namespace_to_total_loss[namespace]
+                total_examples = namespace_to_total_examples[namespace]
+
+                training_loss = total_loss / total_examples
+                training_loss = round(training_loss, 4)
+
+                categorical_accuracy = namespace_to_categorical_accuracy[namespace]
+
+                training_accuracy = categorical_accuracy.get_metric(reset=True)["accuracy"]
+                training_accuracy = round(training_accuracy, 4)
+                logger.info("Step: {}; namespace: {}; training_loss: {}; training_accuracy: {}".format(
+                    idx_step, namespace, training_loss, training_accuracy
+                ))
+                metrics[namespace] = {
+                    "training_loss": training_loss,
+                    "training_accuracy": training_accuracy,
+                }
+            namespace_to_total_loss = defaultdict(float)
+            namespace_to_total_examples = defaultdict(int)
+
+            # evaluation
+            for namespace in namespaces:
+                valid_data_loader = namespace_to_datasets[namespace]["valid_data_loader"]
+
+                model = namespace_to_classifier[namespace]
+                categorical_accuracy = namespace_to_categorical_accuracy[namespace]
+
+                model.eval()
+
+                total_loss = 0
+                total_examples = 0
+                for step, batch in enumerate(valid_data_loader):
+                    input_ids, label_ids = batch
+                    input_ids = input_ids.to(device)
+                    label_ids: torch.LongTensor = label_ids.to(device).long()
+
+                    with torch.no_grad():
+                        logits = model.forward(input_ids)
+                        loss = focal_loss.forward(logits, label_ids.view(-1))
+                        categorical_accuracy(logits, label_ids)
+
+                    total_loss += loss.item()
+                    total_examples += input_ids.size(0)
+
+                evaluation_loss = total_loss / total_examples
+                evaluation_loss = round(evaluation_loss, 4)
+                evaluation_accuracy = categorical_accuracy.get_metric(reset=True)["accuracy"]
+                evaluation_accuracy = round(evaluation_accuracy, 4)
+                logger.info("Step: {}; namespace: {}; evaluation_loss: {}; evaluation_accuracy: {}".format(
+                    idx_step, namespace, evaluation_loss, evaluation_accuracy
+                ))
+                metrics[namespace] = {
+                    "evaluation_loss": evaluation_loss,
+                    "evaluation_accuracy": evaluation_accuracy,
+                }
+
+            # update ratio
+            min_accuracy = min([m["evaluation_accuracy"] for m in metrics.values()])
+            max_accuracy = max([m["evaluation_accuracy"] for m in metrics.values()])
+            width = max_accuracy - min_accuracy
+            for namespace, metric in metrics.items():
+                evaluation_accuracy = metric["evaluation_accuracy"]
+                radio = (max_accuracy - evaluation_accuracy) / width * max_radio
+                radio = int(radio)
+                namespace_to_ratio[namespace] = radio
+
+            msg = "".join(["{}: {}; ".format(k, v) for k, v in namespace_to_ratio.items()])
+            logger.info("namespace to ratio: {}".format(msg))
+
+            # save path
+            step_dir = serialization_dir / "step-{}".format(idx_step)
+            step_dir.mkdir(parents=True, exist_ok=False)
+
+            # save models
+            wave_encoder_filename = step_dir / "wave_encoder.pt"
+            torch.save(wave_encoder.state_dict(), wave_encoder_filename)
+            for namespace in namespaces:
+                cls_head_filename = step_dir / "{}.pt".format(namespace)
+                cls_head = namespace_to_cls_heads[namespace]
+                torch.save(cls_head.state_dict(), cls_head_filename)
+
+            model_list.append(step_dir)
+            if len(model_list) >= args.num_serialized_models_to_keep:
+                model_to_delete: Path = model_list.pop(0)
+                shutil.rmtree(model_to_delete.as_posix())
+
+            # save metric
+            this_accuracy = metrics["global_labels"]["evaluation_accuracy"]
+            if best_accuracy is None:
+                best_idx_step = idx_step
+                best_accuracy = this_accuracy
+            elif metrics["global_labels"]["evaluation_accuracy"] > best_accuracy:
+                best_idx_step = idx_step
+                best_accuracy = this_accuracy
+            else:
+                pass
+
+            metrics_filename = step_dir / "metrics_epoch.json"
+            metrics.update({
+                "idx_step": idx_step,
+                "best_idx_step": best_idx_step,
+            })
+            with open(metrics_filename, "w", encoding="utf-8") as f:
+                json.dump(metrics, f, indent=4, ensure_ascii=False)
+
+            # save best
+            best_dir = serialization_dir / "best"
+            if best_idx_step == idx_step:
+                if best_dir.exists():
+                    shutil.rmtree(best_dir)
+                shutil.copytree(step_dir, best_dir)
+
+            # early stop
+            early_stop_flag = False
+            if best_idx_step == idx_step:
+                patience_count = 0
+            else:
+                patience_count += 1
+            if patience_count >= args.patience:
+                early_stop_flag = True
+
+            # early stop
+            if early_stop_flag:
+                break
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/vm_sound_classification8/stop.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+kill -9 `ps -aef | grep 'vm_sound_classification/bin/python3' | grep -v grep | awk '{print $2}' | sed 's/\n/ /'`

install.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# bash install.sh --stage 2 --stop_stage 2 --system_version centos
+
+
+python_version=3.8.10
+system_version="centos";
+
+verbose=true;
+stage=-1
+stop_stage=0
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+work_dir="$(pwd)"
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  $verbose && echo "stage 1: install python"
+  cd "${work_dir}" || exit 1;
+
+  sh ./script/install_python.sh --python_version "${python_version}" --system_version "${system_version}"
+fi
+
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  $verbose && echo "stage 2: create virtualenv"
+
+  # /usr/local/python-3.6.5/bin/virtualenv vm_sound_classification
+  # source /data/local/bin/vm_sound_classification/bin/activate
+  /usr/local/python-${python_version}/bin/pip3 install virtualenv
+  mkdir -p /data/local/bin
+  cd /data/local/bin || exit 1;
+  /usr/local/python-${python_version}/bin/virtualenv vm_sound_classification
+
+fi

main.py

@@ -0,0 +1,172 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from functools import lru_cache
+import json
+from pathlib import Path
+import platform
+import shutil
+import tempfile
+import zipfile
+
+import gradio as gr
+import numpy as np
+import torch
+
+from project_settings import environment, project_path
+from toolbox.torch.utils.data.vocabulary import Vocabulary
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--examples_dir",
+        default=(project_path / "data/examples").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--trained_model_dir",
+        default=(project_path / "trained_models").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--server_port",
+        default=environment.get("server_port", 7860),
+        type=int
+    )
+    args = parser.parse_args()
+    return args
+
+
+@lru_cache(maxsize=100)
+def load_model(model_file: Path):
+    with zipfile.ZipFile(model_file, "r") as f_zip:
+        out_root = Path(tempfile.gettempdir()) / "vm_sound_classification"
+        if out_root.exists():
+            shutil.rmtree(out_root.as_posix())
+        out_root.mkdir(parents=True, exist_ok=True)
+        f_zip.extractall(path=out_root)
+
+    tgt_path = out_root / model_file.stem
+    jit_model_file = tgt_path / "trace_model.zip"
+    vocab_path = tgt_path / "vocabulary"
+
+    vocabulary = Vocabulary.from_files(vocab_path.as_posix())
+
+    with open(jit_model_file.as_posix(), "rb") as f:
+        model = torch.jit.load(f)
+    model.eval()
+
+    shutil.rmtree(tgt_path)
+
+    d = {
+        "model": model,
+        "vocabulary": vocabulary
+    }
+    return d
+
+
+def click_button(audio: np.ndarray,
+                 model_name: str,
+                 ground_true: str) -> str:
+
+    sample_rate, signal = audio
+
+    model_file = "trained_models/{}.zip".format(model_name)
+    model_file = Path(model_file)
+    d = load_model(model_file)
+
+    model = d["model"]
+    vocabulary = d["vocabulary"]
+
+    inputs = signal / (1 << 15)
+    inputs = torch.tensor(inputs, dtype=torch.float32)
+    inputs = torch.unsqueeze(inputs, dim=0)
+
+    with torch.no_grad():
+        logits = model.forward(inputs)
+        probs = torch.nn.functional.softmax(logits, dim=-1)
+        label_idx = torch.argmax(probs, dim=-1)
+
+    label_idx = label_idx.cpu()
+    probs = probs.cpu()
+
+    label_idx = label_idx.numpy()[0]
+    prob = probs.numpy()[0][label_idx]
+
+    label_str = vocabulary.get_token_from_index(label_idx, namespace="labels")
+
+    return label_str, round(prob, 4)
+
+
+def main():
+    args = get_args()
+
+    examples_dir = Path(args.examples_dir)
+    trained_model_dir = Path(args.trained_model_dir)
+
+    # examples
+    examples = list()
+    for filename in examples_dir.glob("*/*/*.wav"):
+        language = filename.parts[-3]
+        label = filename.parts[-2]
+
+        examples.append([
+            filename.as_posix(),
+            language,
+            label
+        ])
+
+    # models
+    model_choices = list()
+    for filename in trained_model_dir.glob("*.zip"):
+        model_name = filename.stem
+        model_choices.append(model_name)
+
+    # ui
+    brief_description = """
+国际语音智能外呼系统, 电话声音分类. 
+"""
+
+    # ui
+    with gr.Blocks() as blocks:
+        gr.Markdown(value=brief_description)
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                c_audio = gr.Audio(label="audio")
+                with gr.Row():
+                    with gr.Column(scale=3):
+                        c_model_name = gr.Dropdown(choices=model_choices, value=model_choices[0], label="language")
+                    with gr.Column(scale=3):
+                        c_ground_true = gr.Textbox(label="ground_true")
+
+                c_button = gr.Button("run", variant="primary")
+            with gr.Column(scale=3):
+                c_label = gr.Textbox(label="label")
+                c_probability = gr.Number(label="probability")
+
+        gr.Examples(
+            examples,
+            inputs=[c_audio, c_model_name, c_ground_true],
+            outputs=[c_label, c_probability],
+            fn=click_button,
+            examples_per_page=5,
+        )
+
+        c_button.click(
+            click_button,
+            inputs=[c_audio, c_model_name, c_ground_true],
+            outputs=[c_label, c_probability],
+        )
+
+    blocks.queue().launch(
+        share=False if platform.system() == "Windows" else False,
+        server_name="127.0.0.1" if platform.system() == "Windows" else "0.0.0.0",
+        server_port=args.server_port
+    )
+    return
+
+
+if __name__ == "__main__":
+    main()

project_settings.py

@@ -0,0 +1,19 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import os
+from pathlib import Path
+
+from toolbox.os.environment import EnvironmentManager
+
+
+project_path = os.path.abspath(os.path.dirname(__file__))
+project_path = Path(project_path)
+
+environment = EnvironmentManager(
+    path=os.path.join(project_path, "dotenv"),
+    env=os.environ.get("environment", "dev"),
+)
+
+
+if __name__ == '__main__':
+    pass

requirements.txt

@@ -0,0 +1,12 @@
+torch==2.3.0
+torchaudio==2.3.0
+fsspec==2024.5.0
+librosa==0.10.2
+pandas==2.0.3
+openpyxl==3.0.9
+xlrd==1.2.0
+tqdm==4.66.4
+overrides==1.9.0
+pyyaml==6.0.1
+evaluate==0.4.2
+gradio

script/install_nvidia_driver.sh

@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+#GPU驱动安装需要先将原有的显示关闭, 重启机器, 再进行安装.
+#参考链接:
+#https://blog.csdn.net/kingschan/article/details/19033595
+#https://blog.csdn.net/HaixWang/article/details/90408538
+#
+#>>> yum install -y pciutils
+#查看 linux 机器上是否有 GPU
+#lspci |grep -i nvidia
+#
+#>>> lspci |grep -i nvidia
+#00:08.0 3D controller: NVIDIA Corporation TU104GL [Tesla T4] (rev a1)
+#
+#
+#NVIDIA 驱动程序下载
+#先在 pytorch 上查看应该用什么 cuda 版本, 再安装对应的 cuda-toolkit cuda.
+#再根据 gpu 版本下载安装对应的 nvidia 驱动
+#
+## pytorch 版本
+#https://pytorch.org/get-started/locally/
+#
+## CUDA 下载 (好像不需要这个)
+#https://developer.nvidia.com/cuda-toolkit-archive
+#
+## nvidia 驱动
+#https://www.nvidia.cn/Download/index.aspx?lang=cn
+#http://www.nvidia.com/Download/index.aspx
+#
+#在下方的下拉列表中进行选择，针对您的 NVIDIA 产品确定合适的驱动。
+#产品类型:
+#Data Center / Tesla
+#产品系列:
+#T-Series
+#产品家族:
+#Tesla T4
+#操作系统:
+#Linux 64-bit
+#CUDA Toolkit:
+#10.2
+#语言:
+#Chinese (Simpleified)
+#
+#
+#>>> mkdir -p /data/tianxing
+#>>> cd /data/tianxing
+#>>> wget https://cn.download.nvidia.com/tesla/440.118.02/NVIDIA-Linux-x86_64-440.118.02.run
+#>>> sh NVIDIA-Linux-x86_64-440.118.02.run
+#
+## 异常:
+#ERROR: The Nouveau kernel driver is currently in use by your system.  This driver is incompatible with the NVIDIA driver, and must be disabled before proceeding.  Please consult the NVIDIA driver README and your
+#Linux distribution's documentation for details on how to correctly disable the Nouveau kernel driver.
+#[OK]
+#
+#For some distributions, Nouveau can be disabled by adding a file in the modprobe configuration directory.  Would you like nvidia-installer to attempt to create this modprobe file for you?
+#[NO]
+#
+#ERROR: Installation has failed.  Please see the file '/var/log/nvidia-installer.log' for details.  You may find suggestions on fixing installation problems in the README available on the Linux driver download
+#page at www.nvidia.com.
+#[OK]
+#
+## 参考链接:
+#https://blog.csdn.net/kingschan/article/details/19033595
+#
+## 禁用原有的显卡驱动 nouveau
+#>>> echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf
+#>>> sudo dracut --force
+## 重启
+#>>> reboot
+#
+#>>> init 3
+#>>> sh NVIDIA-Linux-x86_64-440.118.02.run
+#
+## 异常
+#ERROR: Unable to find the kernel source tree for the currently running kernel. Please make sure you have installed the kernel source files for your kernel and that they are properly configured; on Red Hat Linux systems, for example, be sure you have the 'kernel-source' or 'kernel-devel' RPM installed. If you know the correct kernel source files are installed, you may specify the kernel source path with the '--kernel-source-path' command line option.
+#[OK]
+#ERROR: Installation has failed.  Please see the file '/var/log/nvidia-installer.log' for details.  You may find suggestions on fixing installation problems in the README available on the Linux driver download
+#page at www.nvidia.com.
+#[OK]
+#
+## 参考链接
+## https://blog.csdn.net/HaixWang/article/details/90408538
+#
+#>>> uname -r
+#3.10.0-1160.49.1.el7.x86_64
+#>>> yum install kernel-devel kernel-headers -y
+#>>> yum info kernel-devel kernel-headers
+#>>> yum install -y "kernel-devel-uname-r == $(uname -r)"
+#>>> yum -y distro-sync
+#
+#>>> sh NVIDIA-Linux-x86_64-440.118.02.run
+#
+## 安装成功
+#WARNING: nvidia-installer was forced to guess the X library path '/usr/lib64' and X module path '/usr/lib64/xorg/modules'; these paths were not queryable from the system.  If X fails to find the NVIDIA X driver
+#module, please install the `pkg-config` utility and the X.Org SDK/development package for your distribution and reinstall the driver.
+#[OK]
+#Install NVIDIA's 32-bit compatibility libraries?
+#[YES]
+#Installation of the kernel module for the NVIDIA Accelerated Graphics Driver for Linux-x86_64 (version 440.118.02) is now complete.
+#[OK]
+#
+#
+## 查看 GPU 使用情况; watch -n 1 -d nvidia-smi 每1秒刷新一次.
+#>>> nvidia-smi
+#Thu Mar  9 12:00:37 2023
+#+-----------------------------------------------------------------------------+
+#| NVIDIA-SMI 440.118.02   Driver Version: 440.118.02   CUDA Version: 10.2     |
+#|-------------------------------+----------------------+----------------------+
+#| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
+#| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
+#|===============================+======================+======================|
+#|   0  Tesla T4            Off  | 00000000:00:08.0 Off |                  Off |
+#| N/A   54C    P0    22W /  70W |      0MiB / 16127MiB |      0%      Default |
+#+-------------------------------+----------------------+----------------------+
+#
+#+-----------------------------------------------------------------------------+
+#| Processes:                                                       GPU Memory |
+#|  GPU       PID   Type   Process name                             Usage      |
+#|=============================================================================|
+#|  No running processes found                                                 |
+#+-----------------------------------------------------------------------------+
+#
+#
+
+# params
+stage=1
+nvidia_driver_filename=https://cn.download.nvidia.com/tesla/440.118.02/NVIDIA-Linux-x86_64-440.118.02.run
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+echo "stage: ${stage}";
+
+yum -y install wget
+yum -y install sudo
+
+if [ ${stage} -eq 0 ]; then
+  mkdir -p /data/dep
+  cd /data/dep || echo 1;
+  wget -P /data/dep ${nvidia_driver_filename}
+
+  echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf
+  sudo dracut --force
+  # 重启
+  reboot
+elif [ ${stage} -eq 1 ]; then
+  init 3
+
+  yum install -y kernel-devel kernel-headers
+  yum info kernel-devel kernel-headers
+  yum install -y "kernel-devel-uname-r == $(uname -r)"
+  yum -y distro-sync
+
+  cd /data/dep || echo 1;
+
+  # 安装时, 需要回车三下.
+  sh NVIDIA-Linux-x86_64-440.118.02.run
+  nvidia-smi
+fi

script/install_python.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+
+# 参数:
+python_version="3.6.5";
+system_version="centos";
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+echo "python_version: ${python_version}";
+echo "system_version: ${system_version}";
+
+
+if [ ${system_version} = "centos" ]; then
+  # 安装 python 开发编译环境
+  yum -y groupinstall "Development tools"
+  yum -y install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel
+  yum install libffi-devel -y
+  yum install -y wget
+  yum install -y make
+
+  mkdir -p /data/dep
+  cd /data/dep || exit 1;
+  if [ ! -e Python-${python_version}.tgz ]; then
+    wget -P /data/dep https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz
+  fi
+
+  cd /data/dep || exit 1;
+  if [ ! -d Python-${python_version} ]; then
+    tar -zxvf Python-${python_version}.tgz
+    cd /data/dep/Python-${python_version} || exit 1;
+  fi
+
+  mkdir /usr/local/python-${python_version}
+  ./configure --prefix=/usr/local/python-${python_version}
+  make && make install
+
+  /usr/local/python-${python_version}/bin/python3 -V
+  /usr/local/python-${python_version}/bin/pip3 -V
+
+  rm -rf /usr/local/bin/python3
+  rm -rf /usr/local/bin/pip3
+  ln -s /usr/local/python-${python_version}/bin/python3 /usr/local/bin/python3
+  ln -s /usr/local/python-${python_version}/bin/pip3 /usr/local/bin/pip3
+
+  python3 -V
+  pip3 -V
+
+elif [ ${system_version} = "ubuntu" ]; then
+  # 安装 python 开发编译环境
+  # https://zhuanlan.zhihu.com/p/506491209
+
+  # 刷新软件包目录
+  sudo apt update
+  # 列出当前可用的更新
+  sudo apt list --upgradable
+  # 如上一步提示有可以更新的项目，则执行更新
+  sudo apt -y upgrade
+  # 安装 GCC 编译器
+  sudo apt install gcc
+  # 检查安装是否成功
+  gcc -v
+
+  # 安装依赖
+  sudo apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev libbz2-dev liblzma-dev sqlite3 libsqlite3-dev tk-dev uuid-dev libgdbm-compat-dev
+
+  mkdir -p /data/dep
+  cd /data/dep || exit 1;
+  if [ ! -e Python-${python_version}.tgz ]; then
+    # sudo wget -P /data/dep https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz
+    sudo wget -P /data/dep https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz
+  fi
+
+  cd /data/dep || exit 1;
+  if [ ! -d Python-${python_version} ]; then
+    # tar -zxvf Python-3.6.5.tgz
+    tar -zxvf Python-${python_version}.tgz
+    # cd /data/dep/Python-3.6.5
+    cd /data/dep/Python-${python_version} || exit 1;
+  fi
+
+  # mkdir /usr/local/python-3.6.5
+  mkdir /usr/local/python-${python_version}
+
+  # 检查依赖与配置编译
+  # sudo ./configure --prefix=/usr/local/python-3.6.5 --enable-optimizations --with-lto --enable-shared
+  sudo ./configure --prefix=/usr/local/python-${python_version} --enable-optimizations --with-lto --enable-shared
+  cpu_count=$(cat /proc/cpuinfo | grep processor | wc -l)
+  # sudo make -j 4
+  sudo make -j "${cpu_count}"
+
+  /usr/local/python-${python_version}/bin/python3 -V
+  /usr/local/python-${python_version}/bin/pip3 -V
+
+  rm -rf /usr/local/bin/python3
+  rm -rf /usr/local/bin/pip3
+  ln -s /usr/local/python-${python_version}/bin/python3 /usr/local/bin/python3
+  ln -s /usr/local/python-${python_version}/bin/pip3 /usr/local/bin/pip3
+
+  python3 -V
+  pip3 -V
+fi

toolbox/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+if __name__ == '__main__':
+    pass

toolbox/json/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/json/misc.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Callable
+
+
+def traverse(js, callback: Callable, *args, **kwargs):
+    if isinstance(js, list):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return result
+    elif isinstance(js, tuple):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return tuple(result)
+    elif isinstance(js, dict):
+        result = dict()
+        for k, v in js.items():
+            k = traverse(k, callback, *args, **kwargs)
+            v = traverse(v, callback, *args, **kwargs)
+            result[k] = v
+        return result
+    elif isinstance(js, int):
+        return callback(js, *args, **kwargs)
+    elif isinstance(js, str):
+        return callback(js, *args, **kwargs)
+    else:
+        return js
+
+
+def demo1():
+    d = {
+        "env": "ppe",
+        "mysql_connect": {
+            "host": "$mysql_connect_host",
+            "port": 3306,
+            "user": "callbot",
+            "password": "NxcloudAI2021!",
+            "database": "callbot_ppe",
+            "charset": "utf8"
+        },
+        "es_connect": {
+            "hosts": ["10.20.251.8"],
+            "http_auth": ["elastic", "ElasticAI2021!"],
+            "port": 9200
+        }
+    }
+
+    def callback(s):
+        if isinstance(s, str) and s.startswith('$'):
+            return s[1:]
+        return s
+
+    result = traverse(d, callback=callback)
+    print(result)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/os/command.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import os
+
+
+class Command(object):
+    custom_command = [
+        "cd"
+    ]
+
+    @staticmethod
+    def _get_cmd(command):
+        command = str(command).strip()
+        if command == "":
+            return None
+        cmd_and_args = command.split(sep=" ")
+        cmd = cmd_and_args[0]
+        args = " ".join(cmd_and_args[1:])
+        return cmd, args
+
+    @classmethod
+    def popen(cls, command):
+        cmd, args = cls._get_cmd(command)
+        if cmd in cls.custom_command:
+            method = getattr(cls, cmd)
+            return method(args)
+        else:
+            resp = os.popen(command)
+            result = resp.read()
+            resp.close()
+            return result
+
+    @classmethod
+    def cd(cls, args):
+        if args.startswith("/"):
+            os.chdir(args)
+        else:
+            pwd = os.getcwd()
+            path = os.path.join(pwd, args)
+            os.chdir(path)
+
+    @classmethod
+    def system(cls, command):
+        return os.system(command)
+
+    def __init__(self):
+        pass
+
+
+def ps_ef_grep(keyword: str):
+    cmd = "ps -ef | grep {}".format(keyword)
+    rows = Command.popen(cmd)
+    rows = str(rows).split("\n")
+    rows = [row for row in rows if row.__contains__(keyword) and not row.__contains__("grep")]
+    return rows
+
+
+if __name__ == "__main__":
+    pass

toolbox/os/environment.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import json
+import os
+
+from dotenv import load_dotenv
+from dotenv.main import DotEnv
+
+from toolbox.json.misc import traverse
+
+
+class EnvironmentManager(object):
+    def __init__(self, path, env, override=False):
+        filename = os.path.join(path, '{}.env'.format(env))
+        self.filename = filename
+
+        load_dotenv(
+            dotenv_path=filename,
+            override=override
+        )
+
+        self._environ = dict()
+
+    def open_dotenv(self, filename: str = None):
+        filename = filename or self.filename
+        dotenv = DotEnv(
+            dotenv_path=filename,
+            stream=None,
+            verbose=False,
+            interpolate=False,
+            override=False,
+            encoding="utf-8",
+        )
+        result = dotenv.dict()
+        return result
+
+    def get(self, key, default=None, dtype=str):
+        result = os.environ.get(key)
+        if result is None:
+            if default is None:
+                result = None
+            else:
+                result = default
+        else:
+            result = dtype(result)
+        self._environ[key] = result
+        return result
+
+
+_DEFAULT_DTYPE_MAP = {
+    'int': int,
+    'float': float,
+    'str': str,
+    'json.loads': json.loads
+}
+
+
+class JsonConfig(object):
+    """
+    将 json 中, 形如 `$float:threshold` 的值, 处理为:
+    从环境变量中查到 threshold, 再将其转换为 float 类型.
+    """
+    def __init__(self, dtype_map: dict = None, environment: EnvironmentManager = None):
+        self.dtype_map = dtype_map or _DEFAULT_DTYPE_MAP
+        self.environment = environment or os.environ
+
+    def sanitize_by_filename(self, filename: str):
+        with open(filename, 'r', encoding='utf-8') as f:
+            js = json.load(f)
+
+        return self.sanitize_by_json(js)
+
+    def sanitize_by_json(self, js):
+        js = traverse(
+            js,
+            callback=self.sanitize,
+            environment=self.environment
+        )
+        return js
+
+    def sanitize(self, string, environment):
+        """支持 $ 符开始的, 环境变量配置"""
+        if isinstance(string, str) and string.startswith('$'):
+            dtype, key = string[1:].split(':')
+            dtype = self.dtype_map[dtype]
+
+            value = environment.get(key)
+            if value is None:
+                raise AssertionError('environment not exist. key: {}'.format(key))
+
+            value = dtype(value)
+            result = value
+        else:
+            result = string
+        return result
+
+
+def demo1():
+    import json
+
+    from project_settings import project_path
+
+    environment = EnvironmentManager(
+        path=os.path.join(project_path, 'server/callbot_server/dotenv'),
+        env='dev',
+    )
+    init_scenes = environment.get(key='init_scenes', dtype=json.loads)
+    print(init_scenes)
+    print(environment._environ)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/other.py

@@ -0,0 +1,9 @@
+import os
+import inspect
+
+
+def pwd():
+    """你在哪个文件调用此函数, 它就会返回那个文件所在的 dir 目标"""
+    frame = inspect.stack()[1]
+    module = inspect.getmodule(frame[0])
+    return os.path.dirname(os.path.abspath(module.__file__))

toolbox/torch/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+if __name__ == '__main__':
+    pass

toolbox/torch/modules/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/torch/modules/gaussian_mixture.py

@@ -0,0 +1,173 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+https://github.com/georgepar/gmmhmm-pytorch/blob/master/gmm.py
+https://github.com/ldeecke/gmm-torch
+"""
+import math
+
+from sklearn import cluster
+import torch
+import torch.nn as nn
+
+
+class GaussianMixtureModel(nn.Module):
+    def __init__(self,
+                 n_mixtures: int,
+                 n_features: int,
+                 init: str = "random",
+                 device: str = 'cpu',
+                 n_iter: int = 1000,
+                 delta: float = 1e-3,
+                 warm_start: bool = False,
+                 ):
+        super(GaussianMixtureModel, self).__init__()
+        self.n_mixtures = n_mixtures
+        self.n_features = n_features
+        self.init = init
+        self.device = device
+        self.n_iter = n_iter
+        self.delta = delta
+        self.warm_start = warm_start
+
+        if init not in ('kmeans', 'random'):
+            raise AssertionError
+
+        self.mu = nn.Parameter(
+            torch.Tensor(n_mixtures, n_features),
+            requires_grad=False,
+        )
+
+        self.sigma = None
+
+        # the weight of each gaussian
+        self.pi = nn.Parameter(
+            torch.Tensor(n_mixtures),
+            requires_grad=False
+        )
+
+        self.converged_ = False
+        self.eps = 1e-6
+        self.delta = delta
+        self.warm_start = warm_start
+        self.n_iter = n_iter
+
+    def reset_sigma(self):
+        raise NotImplementedError
+
+    def estimate_precisions(self):
+        raise NotImplementedError
+
+    def log_prob(self, x):
+        raise NotImplementedError
+
+    def weighted_log_prob(self, x):
+        log_prob = self.log_prob(x)
+        weighted_log_prob = log_prob + torch.log(self.pi)
+        return weighted_log_prob
+
+    def log_likelihood(self, x):
+        weighted_log_prob = self.weighted_log_prob(x)
+        per_sample_log_likelihood = torch.logsumexp(weighted_log_prob, dim=1)
+        log_likelihood = torch.sum(per_sample_log_likelihood)
+        return log_likelihood
+
+    def e_step(self, x):
+        weighted_log_prob = self.weighted_log_prob(x)
+        weighted_log_prob = weighted_log_prob.unsqueeze(dim=-1)
+        log_likelihood = torch.logsumexp(weighted_log_prob, dim=1, keepdim=True)
+        q = weighted_log_prob - log_likelihood
+        return q.squeeze()
+
+    def m_step(self, x, q):
+        x = x.unsqueeze(dim=1)
+        
+        return
+
+    def estimate_mu(self, x, pi, responsibilities):
+        nk = pi * x.size(0)
+        mu = torch.sum(responsibilities * x, dim=0, keepdim=True) / nk
+        return mu
+
+    def estimate_pi(self, x, responsibilities):
+        pi = torch.sum(responsibilities, dim=0, keepdim=True) + self.eps
+        pi = pi / x.size(0)
+        return pi
+
+    def reset_parameters(self, x=None):
+        if self.init == 'random' or x is None:
+            self.mu.normal_()
+            self.reset_sigma()
+            self.pi.fill_(1.0 / self.n_mixtures)
+        elif self.init == 'kmeans':
+            centroids = cluster.KMeans(n_clusters=self.n_mixtures, n_init=1).fit(x).cluster_centers_
+            centroids = torch.tensor(centroids).to(self.device)
+            self.update_(mu=centroids)
+        else:
+            raise NotImplementedError
+
+
+class DiagonalCovarianceGMM(GaussianMixtureModel):
+    def __init__(self,
+                 n_mixtures: int,
+                 n_features: int,
+                 init: str = "random",
+                 device: str = 'cpu',
+                 n_iter: int = 1000,
+                 delta: float = 1e-3,
+                 warm_start: bool = False,
+                 ):
+        super(DiagonalCovarianceGMM, self).__init__(
+            n_mixtures=n_mixtures,
+            n_features=n_features,
+            init=init,
+            device=device,
+            n_iter=n_iter,
+            delta=delta,
+            warm_start=warm_start,
+        )
+        self.sigma = nn.Parameter(
+            torch.Tensor(n_mixtures, n_features), requires_grad=False
+        )
+        self.reset_parameters()
+        self.to(self.device)
+
+    def reset_sigma(self):
+        self.sigma.fill_(1)
+
+    def estimate_precisions(self):
+        return torch.rsqrt(self.sigma)
+
+    def log_prob(self, x):
+        precisions = self.estimate_precisions()
+
+        x = x.unsqueeze(1)
+        mu = self.mu.unsqueeze(0)
+        precisions = precisions.unsqueeze(0)
+
+        # This is outer product
+        exp_term = torch.sum(
+            (mu * mu + x * x - 2 * x * mu) * (precisions ** 2), dim=2, keepdim=True
+        )
+        log_det = torch.sum(torch.log(precisions), dim=2, keepdim=True)
+
+        logp = -0.5 * (self.n_features * torch.log(2 * math.pi) + exp_term) + log_det
+
+        return logp.squeeze()
+
+    def estimate_sigma(self, x, mu, pi, responsibilities):
+        nk = pi * x.size(0)
+        x2 = (responsibilities * x * x).sum(0, keepdim=True) / nk
+        mu2 = mu * mu
+        xmu = (responsibilities * mu * x).sum(0, keepdim=True) / nk
+        sigma = x2 - 2 * xmu + mu2 + self.eps
+
+        return sigma
+
+
+def demo1():
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/torch/modules/highway.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import torch.nn as nn
+
+
+class Highway(nn.Module):
+    """
+    https://arxiv.org/abs/1505.00387
+    [Submitted on 3 May 2015 (v1), last revised 3 Nov 2015 (this version, v2)]
+
+    discuss of Highway and ResNet
+    https://www.zhihu.com/question/279426970
+    """
+    def __init__(self, in_size, out_size):
+        super(Highway, self).__init__()
+        self.H = nn.Linear(in_size, out_size)
+        self.H.bias.data.zero_()
+        self.T = nn.Linear(in_size, out_size)
+        self.T.bias.data.fill_(-1)
+        self.relu = nn.ReLU()
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, inputs):
+        H = self.relu(self.H(inputs))
+        T = self.sigmoid(self.T(inputs))
+        return H * T + inputs * (1.0 - T)
+
+
+if __name__ == '__main__':
+    pass

toolbox/torch/modules/loss.py

@@ -0,0 +1,738 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import math
+from typing import List, Optional
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.modules.loss import _Loss
+from torch.autograd import Variable
+
+
+class ClassBalancedLoss(_Loss):
+    """
+    https://arxiv.org/abs/1901.05555
+    """
+    @staticmethod
+    def demo1():
+        batch_loss: torch.FloatTensor = torch.randn(size=(2, 1), dtype=torch.float32)
+        targets: torch.LongTensor = torch.tensor([1, 2], dtype=torch.long)
+
+        class_balanced_loss = ClassBalancedLoss(
+            num_classes=3,
+            num_samples_each_class=[300, 433, 50],
+            reduction='mean',
+        )
+        loss = class_balanced_loss.forward(batch_loss=batch_loss, targets=targets)
+        print(loss)
+        return
+
+    @staticmethod
+    def demo2():
+        inputs: torch.FloatTensor = torch.randn(size=(2, 3), dtype=torch.float32)
+        targets: torch.LongTensor = torch.tensor([1, 2], dtype=torch.long)
+
+        focal_loss = FocalLoss(
+            num_classes=3,
+            # reduction='mean',
+            # reduction='sum',
+            reduction='none',
+        )
+        batch_loss = focal_loss.forward(inputs, targets)
+        print(batch_loss)
+
+        class_balanced_loss = ClassBalancedLoss(
+            num_classes=3,
+            num_samples_each_class=[300, 433, 50],
+            reduction='mean',
+        )
+        loss = class_balanced_loss.forward(batch_loss=batch_loss, targets=targets)
+        print(loss)
+
+        return
+
+    def __init__(self,
+                 num_classes: int,
+                 num_samples_each_class: List[int],
+                 beta: float = 0.999,
+                 reduction: str = 'mean') -> None:
+        super(ClassBalancedLoss, self).__init__(None, None, reduction)
+
+        effective_num = 1.0 - np.power(beta, num_samples_each_class)
+        weights = (1.0 - beta) / np.array(effective_num)
+        self.weights = weights / np.sum(weights) * num_classes
+
+    def forward(self, batch_loss: torch.FloatTensor, targets: torch.LongTensor):
+        """
+        :param batch_loss: shape=[batch_size, 1]
+        :param targets: shape=[batch_size,]
+        :return:
+        """
+        weights = list()
+        targets = targets.numpy()
+        for target in targets:
+            weights.append([self.weights[target]])
+
+        weights = torch.tensor(weights, dtype=torch.float32)
+        batch_loss = weights * batch_loss
+
+        if self.reduction == 'mean':
+            loss = batch_loss.mean()
+        elif self.reduction == 'sum':
+            loss = batch_loss.sum()
+        else:
+            loss = batch_loss
+        return loss
+
+
+class EqualizationLoss(_Loss):
+    """
+    在图像识别中的, sigmoid 的多标签分类, 且 num_classes 类别数之外有一个 background 背景类别.
+    Equalization Loss
+    https://arxiv.org/abs/2003.05176
+    Equalization Loss v2
+    https://arxiv.org/abs/2012.08548
+    """
+
+    @staticmethod
+    def demo1():
+        logits: torch.FloatTensor = torch.randn(size=(3, 3), dtype=torch.float32)
+        targets: torch.LongTensor = torch.tensor([1, 2, 3], dtype=torch.long)
+
+        equalization_loss = EqualizationLoss(
+            num_samples_each_class=[300, 433, 50],
+            threshold=100,
+            reduction='mean',
+        )
+        loss = equalization_loss.forward(logits=logits, targets=targets)
+        print(loss)
+        return
+
+    def __init__(self,
+                 num_samples_each_class: List[int],
+                 threshold: int = 100,
+                 reduction: str = 'mean') -> None:
+        super(EqualizationLoss, self).__init__(None, None, reduction)
+        self.num_samples_each_class = np.array(num_samples_each_class, dtype=np.int32)
+        self.threshold = threshold
+
+    def forward(self,
+                logits: torch.FloatTensor,
+                targets: torch.LongTensor
+                ):
+        """
+        num_classes + 1 对应于背景类别 background.
+        :param logits: shape=[batch_size, num_classes]
+        :param targets: shape=[batch_size]
+        :return:
+        """
+        batch_size, num_classes = logits.size()
+
+        one_hot_targets = F.one_hot(targets, num_classes=num_classes + 1)
+        one_hot_targets = one_hot_targets[:, :-1]
+
+        exclude = self.exclude_func(
+            num_classes=num_classes,
+            targets=targets
+        )
+        is_tail = self.threshold_func(
+            num_classes=num_classes,
+            num_samples_each_class=self.num_samples_each_class,
+            threshold=self.threshold,
+        )
+
+        weights = 1 - exclude * is_tail * (1 - one_hot_targets)
+
+        batch_loss = F.binary_cross_entropy_with_logits(
+            logits,
+            one_hot_targets.float(),
+            reduction='none'
+        )
+
+        batch_loss = weights * batch_loss
+
+        if self.reduction == 'mean':
+            loss = batch_loss.mean()
+        elif self.reduction == 'sum':
+            loss = batch_loss.sum()
+        else:
+            loss = batch_loss
+
+        loss = loss / num_classes
+        return loss
+
+    @staticmethod
+    def exclude_func(num_classes: int, targets: torch.LongTensor):
+        """
+        最后一个类别是背景 background.
+        :param num_classes: int,
+        :param targets: shape=[batch_size,]
+        :return: weight, shape=[batch_size, num_classes]
+        """
+        batch_size = targets.shape[0]
+        weight = (targets != num_classes).float()
+        weight = weight.view(batch_size, 1).expand(batch_size, num_classes)
+        return weight
+
+    @staticmethod
+    def threshold_func(num_classes: int, num_samples_each_class: np.ndarray, threshold: int):
+        """
+        :param num_classes: int,
+        :param num_samples_each_class: shape=[num_classes]
+        :param threshold: int,
+        :return: weight, shape=[1, num_classes]
+        """
+        weight = torch.zeros(size=(num_classes,))
+        weight[num_samples_each_class < threshold] = 1
+        weight = torch.unsqueeze(weight, dim=0)
+        return weight
+
+
+class FocalLoss(_Loss):
+    """
+    https://arxiv.org/abs/1708.02002
+    """
+    @staticmethod
+    def demo1(self):
+        inputs: torch.FloatTensor = torch.randn(size=(2, 3), dtype=torch.float32)
+        targets: torch.LongTensor = torch.tensor([1, 2], dtype=torch.long)
+
+        focal_loss = FocalLoss(
+            num_classes=3,
+            reduction='mean',
+            # reduction='sum',
+            # reduction='none',
+        )
+        loss = focal_loss.forward(inputs, targets)
+        print(loss)
+        return
+
+    def __init__(self,
+                 num_classes: int,
+                 alpha: List[float] = None,
+                 gamma: int = 2,
+                 reduction: str = 'mean',
+                 inputs_logits: bool = True) -> None:
+        """
+        :param num_classes:
+        :param alpha:
+        :param gamma:
+        :param reduction: (`none`, `mean`, `sum`) available.
+        :param inputs_logits: if False, the inputs should be probs.
+        """
+        super(FocalLoss, self).__init__(None, None, reduction)
+        if alpha is None:
+            self.alpha = torch.ones(num_classes, 1)
+        else:
+            self.alpha = torch.tensor(alpha, dtype=torch.float32)
+        self.gamma = gamma
+        self.num_classes = num_classes
+        self.inputs_logits = inputs_logits
+
+    def forward(self,
+                inputs: torch.FloatTensor,
+                targets: torch.LongTensor):
+        """
+        :param inputs: logits, shape=[batch_size, num_classes]
+        :param targets: shape=[batch_size,]
+        :return:
+        """
+        batch_size, num_classes = inputs.shape
+
+        if self.inputs_logits:
+            probs = F.softmax(inputs, dim=-1)
+        else:
+            probs = inputs
+
+        # class_mask = inputs.data.new(batch_size, num_classes).fill_(0)
+        class_mask = torch.zeros(size=(batch_size, num_classes), dtype=inputs.dtype, device=inputs.device)
+        # class_mask = Variable(class_mask)
+        ids = targets.view(-1, 1)
+        class_mask.scatter_(1, ids.data, 1.)
+
+        if inputs.is_cuda and not self.alpha.is_cuda:
+            self.alpha = self.alpha.cuda()
+        alpha = self.alpha[ids.data.view(-1)]
+
+        probs = (probs * class_mask).sum(1).view(-1, 1)
+
+        log_p = probs.log()
+
+        batch_loss = -alpha*(torch.pow((1-probs), self.gamma))*log_p
+
+        if self.reduction == 'mean':
+            loss = batch_loss.mean()
+        elif self.reduction == 'sum':
+            loss = batch_loss.sum()
+        else:
+            loss = batch_loss
+        return loss
+
+
+class HingeLoss(_Loss):
+    @staticmethod
+    def demo1():
+        inputs: torch.FloatTensor = torch.randn(size=(2, 3), dtype=torch.float32)
+        targets: torch.LongTensor = torch.tensor([1, 2], dtype=torch.long)
+
+        hinge_loss = HingeLoss(
+            margin_list=[300, 433, 50],
+            reduction='mean',
+        )
+        loss = hinge_loss.forward(inputs=inputs, targets=targets)
+        print(loss)
+        return
+
+    def __init__(self,
+                 margin_list: List[float],
+                 max_margin: float = 0.5,
+                 scale: float = 1.0,
+                 weight: Optional[torch.Tensor] = None,
+                 reduction: str = 'mean') -> None:
+        super(HingeLoss, self).__init__(None, None, reduction)
+
+        self.max_margin = max_margin
+        self.scale = scale
+        self.weight = weight
+
+        margin_list = np.array(margin_list)
+        margin_list = margin_list * (max_margin / np.max(margin_list))
+        self.margin_list = torch.tensor(margin_list, dtype=torch.float32)
+
+    def forward(self,
+                inputs: torch.FloatTensor,
+                targets: torch.LongTensor
+                ):
+        """
+        :param inputs: logits, shape=[batch_size, num_classes]
+        :param targets: shape=[batch_size,]
+        :return:
+        """
+        batch_size, num_classes = inputs.shape
+        one_hot_targets = F.one_hot(targets, num_classes=num_classes)
+        margin_list = torch.unsqueeze(self.margin_list, dim=0)
+
+        batch_margin = torch.sum(margin_list * one_hot_targets, dim=-1)
+        batch_margin = torch.unsqueeze(batch_margin, dim=-1)
+        inputs_margin = inputs - batch_margin
+
+        # 将类别对应的 logits 值减小一点, 以形成 margin 边界.
+        logits = torch.where(one_hot_targets > 0, inputs_margin, inputs)
+
+        loss = F.cross_entropy(
+            input=self.scale * logits,
+            target=targets,
+            weight=self.weight,
+            reduction=self.reduction,
+        )
+        return loss
+
+
+class HingeLinear(nn.Module):
+    """
+    use this instead of `HingeLoss`, then you can combine it with `FocalLoss` or others.
+    """
+    def __init__(self,
+                 margin_list: List[float],
+                 max_margin: float = 0.5,
+                 scale: float = 1.0,
+                 weight: Optional[torch.Tensor] = None
+                 ) -> None:
+        super(HingeLinear, self).__init__()
+
+        self.max_margin = max_margin
+        self.scale = scale
+        self.weight = weight
+
+        margin_list = np.array(margin_list)
+        margin_list = margin_list * (max_margin / np.max(margin_list))
+        self.margin_list = torch.tensor(margin_list, dtype=torch.float32)
+
+    def forward(self,
+                inputs: torch.FloatTensor,
+                targets: torch.LongTensor
+                ):
+        """
+        :param inputs: logits, shape=[batch_size, num_classes]
+        :param targets: shape=[batch_size,]
+        :return:
+        """
+        if self.training and targets is not None:
+            batch_size, num_classes = inputs.shape
+            one_hot_targets = F.one_hot(targets, num_classes=num_classes)
+            margin_list = torch.unsqueeze(self.margin_list, dim=0)
+
+            batch_margin = torch.sum(margin_list * one_hot_targets, dim=-1)
+            batch_margin = torch.unsqueeze(batch_margin, dim=-1)
+            inputs_margin = inputs - batch_margin
+
+            # 将类别对应的 logits 值减小一点, 以形成 margin 边界.
+            logits = torch.where(one_hot_targets > 0, inputs_margin, inputs)
+            logits = logits * self.scale
+        else:
+            logits = inputs
+        return logits
+
+
+class LDAMLoss(_Loss):
+    """
+    https://arxiv.org/abs/1906.07413
+    """
+    @staticmethod
+    def demo1():
+        inputs: torch.FloatTensor = torch.randn(size=(2, 3), dtype=torch.float32)
+        targets: torch.LongTensor = torch.tensor([1, 2], dtype=torch.long)
+
+        ldam_loss = LDAMLoss(
+            num_samples_each_class=[300, 433, 50],
+            reduction='mean',
+        )
+        loss = ldam_loss.forward(inputs=inputs, targets=targets)
+        print(loss)
+        return
+
+    def __init__(self,
+                 num_samples_each_class: List[int],
+                 max_margin: float = 0.5,
+                 scale: float = 30.0,
+                 weight: Optional[torch.Tensor] = None,
+                 reduction: str = 'mean') -> None:
+        super(LDAMLoss, self).__init__(None, None, reduction)
+
+        margin_list = np.power(num_samples_each_class, -0.25)
+        margin_list = margin_list * (max_margin / np.max(margin_list))
+
+        self.num_samples_each_class = num_samples_each_class
+        self.margin_list = torch.tensor(margin_list, dtype=torch.float32)
+        self.scale = scale
+        self.weight = weight
+
+    def forward(self,
+                inputs: torch.FloatTensor,
+                targets: torch.LongTensor
+                ):
+        """
+        :param inputs: logits, shape=[batch_size, num_classes]
+        :param targets: shape=[batch_size,]
+        :return:
+        """
+        batch_size, num_classes = inputs.shape
+        one_hot_targets = F.one_hot(targets, num_classes=num_classes)
+        margin_list = torch.unsqueeze(self.margin_list, dim=0)
+
+        batch_margin = torch.sum(margin_list * one_hot_targets, dim=-1)
+        batch_margin = torch.unsqueeze(batch_margin, dim=-1)
+        inputs_margin = inputs - batch_margin
+
+        # 将类别对应的 logits 值减小一点, 以形成 margin 边界.
+        logits = torch.where(one_hot_targets > 0, inputs_margin, inputs)
+
+        loss = F.cross_entropy(
+            input=self.scale * logits,
+            target=targets,
+            weight=self.weight,
+            reduction=self.reduction,
+        )
+        return loss
+
+
+class NegativeEntropy(_Loss):
+    def __init__(self,
+                 reduction: str = 'mean',
+                 inputs_logits: bool = True) -> None:
+        super(NegativeEntropy, self).__init__(None, None, reduction)
+        self.inputs_logits = inputs_logits
+
+    def forward(self,
+                inputs: torch.FloatTensor,
+                targets: torch.LongTensor):
+        if self.inputs_logits:
+            probs = F.softmax(inputs, dim=-1)
+            log_probs = torch.nn.functional.log_softmax(probs, dim=-1)
+        else:
+            probs = inputs
+            log_probs = torch.log(probs)
+
+        weighted_negative_likelihood = - log_probs * probs
+
+        loss = - weighted_negative_likelihood.sum()
+        return loss
+
+
+class LargeMarginSoftMaxLoss(_Loss):
+    """
+    Alias: L-Softmax
+
+    https://arxiv.org/abs/1612.02295
+    https://github.com/wy1iu/LargeMargin_Softmax_Loss
+    https://github.com/amirhfarzaneh/lsoftmax-pytorch/blob/master/lsoftmax.py
+
+    参考链接:
+    https://www.jianshu.com/p/06cc3f84aa85
+
+    论文认为, softmax 和 cross entropy 的组合, 没有明确鼓励对特征进行判别学习.
+
+    """
+    def __init__(self,
+                 reduction: str = 'mean') -> None:
+        super(LargeMarginSoftMaxLoss, self).__init__(None, None, reduction)
+
+
+class AngularSoftMaxLoss(_Loss):
+    """
+    Alias: A-Softmax
+
+    https://arxiv.org/abs/1704.08063
+
+    https://github.com/woshildh/a-softmax_pytorch/blob/master/a_softmax.py
+
+    参考链接:
+    https://www.jianshu.com/p/06cc3f84aa85
+
+    好像作者认为人脸是一个球面, 所以将向量转换到一个球面上是有帮助的.
+    """
+    def __init__(self,
+                 reduction: str = 'mean') -> None:
+        super(AngularSoftMaxLoss, self).__init__(None, None, reduction)
+
+
+class AdditiveMarginSoftMax(_Loss):
+    """
+    Alias: AM-Softmax
+
+    https://arxiv.org/abs/1801.05599
+
+    Large Margin Cosine Loss
+    https://arxiv.org/abs/1801.09414
+
+    参考链接:
+    https://www.jianshu.com/p/06cc3f84aa85
+
+    说明:
+    相对于普通的 对 logits 做 softmax,
+    它将真实标签对应的 logit 值减去 m, 来让模型它该值调整得更大一些.
+    另外, 它还将每个 logits 乘以 s, 这可以控制各 logits 之间的相对大小.
+    根 HingeLoss 有点像.
+    """
+    def __init__(self,
+                 reduction: str = 'mean') -> None:
+        super(AdditiveMarginSoftMax, self).__init__(None, None, reduction)
+
+
+class AdditiveAngularMarginSoftMax(_Loss):
+    """
+    Alias: ArcFace, AAM-Softmax
+
+    ArcFace: Additive Angular Margin Loss for Deep Face Recognition
+    https://arxiv.org/abs/1801.07698
+
+    参考代码:
+    https://github.com/huangkeju/AAMSoftmax-OpenMax/blob/main/AAMSoftmax%2BOvA/metrics.py
+
+    """
+    @staticmethod
+    def demo1():
+        """
+        角度与数值转换
+        pi / 180 代表 1 度,
+        pi / 180 = 0.01745
+        """
+
+        # 度数转数值
+        degree = 10
+        result = degree * math.pi / 180
+        print(result)
+
+        # 数值转数度
+        radian = 0.2
+        result = radian / (math.pi / 180)
+        print(result)
+
+        return
+
+    def __init__(self,
+                 hidden_size: int,
+                 num_labels: int,
+                 margin: float = 0.2,
+                 scale: float = 10.0,
+                 ):
+        """
+        :param hidden_size:
+        :param num_labels:
+        :param margin: 建议取值角度为 [10, 30], 对应的数值为 [0.1745, 0.5236]
+        :param scale:
+        """
+        super(AdditiveAngularMarginSoftMax, self).__init__()
+        self.margin = margin
+        self.scale = scale
+        self.weight = torch.nn.Parameter(torch.FloatTensor(num_labels, hidden_size), requires_grad=True)
+        nn.init.xavier_uniform_(self.weight)
+
+        self.cos_margin = math.cos(self.margin)
+        self.sin_margin = math.sin(self.margin)
+
+        # sin(a-b) = sin(a)cos(b) - cos(a)sin(b)
+        # sin(pi - a) = sin(a)
+
+        self.loss = nn.CrossEntropyLoss()
+
+    def forward(self,
+                inputs: torch.Tensor,
+                label: torch.LongTensor = None
+                ):
+        """
+        :param inputs: shape=[batch_size, ..., hidden_size]
+        :param label:
+        :return: logits
+        """
+        x = F.normalize(inputs)
+        weight = F.normalize(self.weight)
+        cosine = F.linear(x, weight)
+
+        if self.training:
+
+            # sin^2  + cos^2 = 1
+            sine = torch.sqrt((1.0 - torch.mul(cosine, cosine)).clamp(0, 1))
+
+            # cos(a+b) = cos(a)cos(b) - sin(a)sin(b)
+            cosine_theta_margin = cosine * self.cos_margin - sine * self.sin_margin
+
+            # when the `cosine > - self.cos_margin` there is enough space to add margin on theta.
+            cosine_theta_margin = torch.where(cosine > - self.cos_margin, cosine_theta_margin, cosine - (self.margin * self.sin_margin))
+
+            one_hot = torch.zeros_like(cosine)
+            one_hot.scatter_(1, label.view(-1, 1), 1)
+
+            #
+            logits = torch.where(one_hot == 1, cosine_theta_margin, cosine)
+            logits = logits * self.scale
+        else:
+            logits = cosine
+
+        loss = self.loss(logits, label)
+        # prec1 = accuracy(output.detach(), label.detach(), topk=(1,))[0]
+        return loss
+
+
+class AdditiveAngularMarginLinear(nn.Module):
+    """
+    Alias: ArcFace, AAM-Softmax
+
+    ArcFace: Additive Angular Margin Loss for Deep Face Recognition
+    https://arxiv.org/abs/1801.07698
+
+    参考代码:
+    https://github.com/huangkeju/AAMSoftmax-OpenMax/blob/main/AAMSoftmax%2BOvA/metrics.py
+
+    """
+    @staticmethod
+    def demo1():
+        """
+        角度与数值转换
+        pi / 180 代表 1 度,
+        pi / 180 = 0.01745
+        """
+
+        # 度数转数值
+        degree = 10
+        result = degree * math.pi / 180
+        print(result)
+
+        # 数值转数度
+        radian = 0.2
+        result = radian / (math.pi / 180)
+        print(result)
+
+        return
+
+    @staticmethod
+    def demo2():
+
+        return
+
+    def __init__(self,
+                 hidden_size: int,
+                 num_labels: int,
+                 margin: float = 0.2,
+                 scale: float = 10.0,
+                 ):
+        """
+        :param hidden_size:
+        :param num_labels:
+        :param margin: 建议取值角度为 [10, 30], 对应的数值为 [0.1745, 0.5236]
+        :param scale:
+        """
+        super(AdditiveAngularMarginLinear, self).__init__()
+        self.margin = margin
+        self.scale = scale
+        self.weight = torch.nn.Parameter(torch.FloatTensor(num_labels, hidden_size), requires_grad=True)
+        nn.init.xavier_uniform_(self.weight)
+
+        self.cos_margin = math.cos(self.margin)
+        self.sin_margin = math.sin(self.margin)
+
+        # sin(a-b) = sin(a)cos(b) - cos(a)sin(b)
+        # sin(pi - a) = sin(a)
+
+    def forward(self,
+                inputs: torch.Tensor,
+                targets: torch.LongTensor = None
+                ):
+        """
+        :param inputs: shape=[batch_size, ..., hidden_size]
+        :param targets:
+        :return: logits
+        """
+        x = F.normalize(inputs)
+        weight = F.normalize(self.weight)
+        cosine = F.linear(x, weight)
+
+        if self.training and targets is not None:
+            # sin^2  + cos^2 = 1
+            sine = torch.sqrt((1.0 - torch.mul(cosine, cosine)).clamp(0, 1))
+
+            # cos(a+b) = cos(a)cos(b) - sin(a)sin(b)
+            cosine_theta_margin = cosine * self.cos_margin - sine * self.sin_margin
+
+            # when the `cosine > - self.cos_margin` there is enough space to add margin on theta.
+            cosine_theta_margin = torch.where(cosine > - self.cos_margin, cosine_theta_margin, cosine - (self.margin * self.sin_margin))
+
+            one_hot = torch.zeros_like(cosine)
+            one_hot.scatter_(1, targets.view(-1, 1), 1)
+
+            logits = torch.where(one_hot == 1, cosine_theta_margin, cosine)
+            logits = logits * self.scale
+        else:
+            logits = cosine
+        return logits
+
+
+def demo1():
+    HingeLoss.demo1()
+    return
+
+
+def demo2():
+    AdditiveAngularMarginSoftMax.demo1()
+
+    inputs = torch.ones(size=(2, 5), dtype=torch.float32)
+    label: torch.LongTensor = torch.tensor(data=[0, 1], dtype=torch.long)
+
+    aam_softmax = AdditiveAngularMarginSoftMax(
+        hidden_size=5,
+        num_labels=2,
+        margin=1,
+        scale=1
+    )
+
+    outputs = aam_softmax.forward(inputs, label)
+    print(outputs)
+
+    return
+
+
+if __name__ == '__main__':
+    # demo1()
+    demo2()

toolbox/torch/training/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/torch/training/metrics/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/torch/training/metrics/categorical_accuracy.py

@@ -0,0 +1,82 @@
+from typing import Optional
+
+from overrides import overrides
+import torch
+
+
+class CategoricalAccuracy(object):
+    def __init__(self, top_k: int = 1, tie_break: bool = False) -> None:
+        if top_k > 1 and tie_break:
+            raise AssertionError("Tie break in Categorical Accuracy "
+                                 "can be done only for maximum (top_k = 1)")
+        if top_k <= 0:
+            raise AssertionError("top_k passed to Categorical Accuracy must be > 0")
+        self._top_k = top_k
+        self._tie_break = tie_break
+        self.correct_count = 0.
+        self.total_count = 0.
+
+    def __call__(self,
+                 predictions: torch.Tensor,
+                 gold_labels: torch.Tensor,
+                 mask: Optional[torch.Tensor] = None):
+
+        # predictions, gold_labels, mask = self.unwrap_to_tensors(predictions, gold_labels, mask)
+
+        # Some sanity checks.
+        num_classes = predictions.size(-1)
+        if gold_labels.dim() != predictions.dim() - 1:
+            raise AssertionError("gold_labels must have dimension == predictions.size() - 1 but "
+                                 "found tensor of shape: {}".format(predictions.size()))
+        if (gold_labels >= num_classes).any():
+            raise AssertionError("A gold label passed to Categorical Accuracy contains an id >= {}, "
+                                 "the number of classes.".format(num_classes))
+
+        predictions = predictions.view((-1, num_classes))
+        gold_labels = gold_labels.view(-1).long()
+        if not self._tie_break:
+            # Top K indexes of the predictions (or fewer, if there aren't K of them).
+            # Special case topk == 1, because it's common and .max() is much faster than .topk().
+            if self._top_k == 1:
+                top_k = predictions.max(-1)[1].unsqueeze(-1)
+            else:
+                top_k = predictions.topk(min(self._top_k, predictions.shape[-1]), -1)[1]
+
+            # This is of shape (batch_size, ..., top_k).
+            correct = top_k.eq(gold_labels.unsqueeze(-1)).float()
+        else:
+            # prediction is correct if gold label falls on any of the max scores. distribute score by tie_counts
+            max_predictions = predictions.max(-1)[0]
+            max_predictions_mask = predictions.eq(max_predictions.unsqueeze(-1))
+            # max_predictions_mask is (rows X num_classes) and gold_labels is (batch_size)
+            # ith entry in gold_labels points to index (0-num_classes) for ith row in max_predictions
+            # For each row check if index pointed by gold_label is was 1 or not (among max scored classes)
+            correct = max_predictions_mask[torch.arange(gold_labels.numel()).long(), gold_labels].float()
+            tie_counts = max_predictions_mask.sum(-1)
+            correct /= tie_counts.float()
+            correct.unsqueeze_(-1)
+
+        if mask is not None:
+            correct *= mask.view(-1, 1).float()
+            self.total_count += mask.sum()
+        else:
+            self.total_count += gold_labels.numel()
+        self.correct_count += correct.sum()
+
+    def get_metric(self, reset: bool = False):
+        """
+        Returns
+        -------
+        The accumulated accuracy.
+        """
+        if self.total_count > 1e-12:
+            accuracy = float(self.correct_count) / float(self.total_count)
+        else:
+            accuracy = 0.0
+        if reset:
+            self.reset()
+        return {'accuracy': accuracy}
+
+    def reset(self):
+        self.correct_count = 0.0
+        self.total_count = 0.0

toolbox/torch/training/metrics/verbose_categorical_accuracy.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Dict, List, Optional
+
+import numpy as np
+import torch
+
+
+class CategoricalAccuracyVerbose(object):
+    def __init__(self,
+                 index_to_token: Dict[int, str],
+                 label_namespace: str = "labels",
+                 top_k: int = 1,
+                 ) -> None:
+        if top_k <= 0:
+            raise AssertionError("top_k passed to Categorical Accuracy must be > 0")
+        self._index_to_token = index_to_token
+        self._label_namespace = label_namespace
+        self._top_k = top_k
+        self.correct_count = 0.
+        self.total_count = 0.
+        self.label_correct_count = dict()
+        self.label_total_count = dict()
+
+    def __call__(self,
+                 predictions: torch.Tensor,
+                 gold_labels: torch.Tensor,
+                 mask: Optional[torch.Tensor] = None):
+        num_classes = predictions.size(-1)
+        if gold_labels.dim() != predictions.dim() - 1:
+            raise AssertionError("gold_labels must have dimension == predictions.size() - 1 but "
+                                     "found tensor of shape: {}".format(predictions.size()))
+        if (gold_labels >= num_classes).any():
+            raise AssertionError("A gold label passed to Categorical Accuracy contains an id >= {}, "
+                                     "the number of classes.".format(num_classes))
+
+        predictions = predictions.view((-1, num_classes))
+        gold_labels = gold_labels.view(-1).long()
+
+        # Top K indexes of the predictions (or fewer, if there aren't K of them).
+        # Special case topk == 1, because it's common and .max() is much faster than .topk().
+        if self._top_k == 1:
+            top_k = predictions.max(-1)[1].unsqueeze(-1)
+        else:
+            top_k = predictions.topk(min(self._top_k, predictions.shape[-1]), -1)[1]
+
+        # This is of shape (batch_size, ..., top_k).
+        correct = top_k.eq(gold_labels.unsqueeze(-1)).float()
+
+        if mask is not None:
+            correct *= mask.view(-1, 1).float()
+            self.total_count += mask.sum()
+        else:
+            self.total_count += gold_labels.numel()
+        self.correct_count += correct.sum()
+
+        labels: List[int] = np.unique(gold_labels.cpu().numpy()).tolist()
+        for label in labels:
+            label_mask = (gold_labels == label)
+
+            label_correct = correct * label_mask.view(-1, 1).float()
+            label_correct = int(label_correct.sum())
+            label_count = int(label_mask.sum())
+
+            label_str = self._index_to_token[label]
+            if label_str in self.label_correct_count:
+                self.label_correct_count[label_str] += label_correct
+            else:
+                self.label_correct_count[label_str] = label_correct
+
+            if label_str in self.label_total_count:
+                self.label_total_count[label_str] += label_count
+            else:
+                self.label_total_count[label_str] = label_count
+
+    def get_metric(self, reset: bool = False):
+        """
+        Returns
+        -------
+        The accumulated accuracy.
+        """
+        result = dict()
+        if self.total_count > 1e-12:
+            accuracy = float(self.correct_count) / float(self.total_count)
+        else:
+            accuracy = 0.0
+        result['accuracy'] = accuracy
+
+        for label in self.label_total_count.keys():
+            total = self.label_total_count[label]
+            correct = self.label_correct_count.get(label, 0.0)
+            label_accuracy = correct / total
+            result[label] = label_accuracy
+
+        if reset:
+            self.reset()
+        return result
+
+    def reset(self):
+        self.correct_count = 0.0
+        self.total_count = 0.0
+        self.label_correct_count = dict()
+        self.label_total_count = dict()
+
+
+def demo1():
+
+    categorical_accuracy_verbose = CategoricalAccuracyVerbose(
+        index_to_token={0: '0', 1: '1'},
+        top_k=2,
+    )
+
+    predictions = torch.randn(size=(2, 3), dtype=torch.float32)
+    gold_labels = torch.ones(size=(2,), dtype=torch.long)
+    # print(predictions)
+    # print(gold_labels)
+
+    categorical_accuracy_verbose(
+        predictions=predictions,
+        gold_labels=gold_labels,
+    )
+    metric = categorical_accuracy_verbose.get_metric()
+    print(metric)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/torch/training/trainer/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+if __name__ == '__main__':
+    pass

toolbox/torch/training/trainer/trainer.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+if __name__ == '__main__':
+    pass

toolbox/torch/utils/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+if __name__ == '__main__':
+    pass

toolbox/torchaudio/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+if __name__ == '__main__':
+    pass

toolbox/torchaudio/configuration_utils.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import copy
+import os
+from typing import Any, Dict, Union
+
+import yaml
+
+
+CONFIG_FILE = "config.yaml"
+
+
+class PretrainedConfig(object):
+    def __init__(self, **kwargs):
+        pass
+
+    @classmethod
+    def _dict_from_yaml_file(cls, yaml_file: Union[str, os.PathLike]):
+        with open(yaml_file, encoding="utf-8") as f:
+            config_dict = yaml.safe_load(f)
+        return config_dict
+
+    @classmethod
+    def get_config_dict(
+        cls, pretrained_model_name_or_path: Union[str, os.PathLike]
+    ) -> Dict[str, Any]:
+        if os.path.isdir(pretrained_model_name_or_path):
+            config_file = os.path.join(pretrained_model_name_or_path, CONFIG_FILE)
+        else:
+            config_file = pretrained_model_name_or_path
+        config_dict = cls._dict_from_yaml_file(config_file)
+        return config_dict
+
+    @classmethod
+    def from_dict(cls, config_dict: Dict[str, Any], **kwargs):
+        for k, v in kwargs.items():
+            if k in config_dict.keys():
+                config_dict[k] = v
+        config = cls(**config_dict)
+        return config
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        pretrained_model_name_or_path: Union[str, os.PathLike],
+        **kwargs,
+    ):
+        config_dict = cls.get_config_dict(pretrained_model_name_or_path)
+        return cls.from_dict(config_dict, **kwargs)
+
+    def to_dict(self):
+        output = copy.deepcopy(self.__dict__)
+        return output
+
+    def to_yaml_file(self, yaml_file_path: Union[str, os.PathLike]):
+        config_dict = self.to_dict()
+
+        with open(yaml_file_path, "w", encoding="utf-8") as writer:
+            yaml.safe_dump(config_dict, writer)
+
+
+if __name__ == '__main__':
+    pass