jjiang4/acesinger_svs_train_visinger2

ESPnet3 gansvs model

Packed model bundle generated from egs3/acesinger/svs.

Model

• Repository: jjiang4/acesinger_svs_train_visinger2
• Recipe: egs3/acesinger/svs
• Corpus: acesinger
• System: gansvs
• Creator: jjiang4
• Created: 2026-09-13T20:30:01
• Branch: acesinger
• Git: 5fd502819d (dirty)
• Origin: git@github.com:Ting-Justin-Jiang/espnet.git

Model summary

• Class: ESPnetGANSVSModel
• Total parameters: 107,319,495
• Learnable parameters: 107,319,495 (100.0%)
• Non-trainable parameters: 0
• Parameter size: 429.28 MB
• Buffers: 4,591,030 (18.36 MB)
• Modules: 1,297 total, 916 leaf
• DType composition: torch.float32(104.3%), torch.float64(0.0%)

Usage

from espnet3.publication import InferenceModel

model = InferenceModel.from_pretrained("jjiang4/acesinger_svs_train_visinger2", trust_user_code=True)
# `sample` is one item of the recipe dataset built with `inference: true`:
# the music score under "text" ({"label": (times, phones), "score": (tempo,
# notes)}) plus the singer id "sids". `result["wav"]` is the waveform.
result = model(sample)

Packaging

• Bundle: model_pack
• Exp dir: ./exp/train_visinger2
• Strategy: copy experiment outputs; include extra recipe assets; apply exclude filters

Results

dataset	log_f0_rmse	mcd	semitone_acc	vuv_err
test	0.18274712676728574	6.103883153889555	0.5672507490139542	0.09170632745165584

Training config

expand

num_device: 4
num_nodes: 1
task: espnet2.tasks.gan_svs.GANSVSTask
recipe_dir: .
data_dir: ./data
exp_tag: train_visinger2
exp_dir: ./exp/train_visinger2
stats_dir: ./exp/train_visinger2/stats
inference_dir: ./exp/train_visinger2/inference
dataset:
  _target_: espnet3.components.data.data_organizer.DataOrganizer
  _recursive_: false
  recipe_dir: .
  train:
  - data_src_args:
      split: train
      manifest_path: ./data/manifest_filtered/train.tsv
      collect_feats_dir: ./exp/train_visinger2/stats/train/collect_feats
  valid:
  - data_src_args:
      split: valid
      manifest_path: ./data/manifest_filtered/valid.tsv
      collect_feats_dir: ./exp/train_visinger2/stats/valid/collect_feats
  test: null
  preprocessor:
    _target_: espnet2.train.preprocessor.SVSPreprocessor
    train: true
    token_type: phn
    token_list: ./data/token_list/phn_none/tokens.txt
    fs: 44100
    hop_length: 512
    _convert_: all
  _convert_: all
model:
  token_list: ./data/token_list/phn_none/tokens.txt
  feats_extract: fbank
  feats_extract_conf:
    fs: 44100
    n_fft: 2048
    hop_length: 512
    win_length: 2048
    fmin: 80
    fmax: 22050
    n_mels: 80
  normalize: global_mvn
  normalize_conf:
    stats_file: ./exp/train_visinger2/stats/train/feats_stats.npz
  score_feats_extract: syllable_score_feats
  score_feats_extract_conf:
    fs: 44100
    n_fft: 2048
    win_length: 2048
    hop_length: 512
  pitch_extract: dio
  pitch_extract_conf:
    fs: 44100
    n_fft: 2048
    hop_length: 512
    f0max: 810
    f0min: 80
    use_token_averaged_f0: false
    use_log_f0: false
  pitch_normalize: null
  svs: vits
  svs_conf:
    generator_type: visinger2
    vocoder_generator_type: visinger2
    generator_params:
      hidden_channels: 192
      spks: 31
      global_channels: 256
      segment_size: 20
      text_encoder_attention_heads: 2
      text_encoder_ffn_expand: 4
      text_encoder_blocks: 6
      text_encoder_positionwise_layer_type: conv1d
      text_encoder_positionwise_conv_kernel_size: 3
      text_encoder_positional_encoding_layer_type: rel_pos
      text_encoder_self_attention_layer_type: rel_selfattn
      text_encoder_activation_type: swish
      text_encoder_normalize_before: true
      text_encoder_dropout_rate: 0.1
      text_encoder_positional_dropout_rate: 0.0
      text_encoder_attention_dropout_rate: 0.1
      use_macaron_style_in_text_encoder: true
      use_conformer_conv_in_text_encoder: false
      text_encoder_conformer_kernel_size: -1
      decoder_kernel_size: 7
      decoder_channels: 256
      decoder_upsample_scales:
      - 8
      - 8
      - 4
      - 2
      decoder_upsample_kernel_sizes:
      - 16
      - 16
      - 8
      - 4
      n_harmonic: 64
      decoder_resblock_kernel_sizes:
      - 3
      - 7
      - 11
      decoder_resblock_dilations:
      - - 1
        - 3
        - 5
      - - 1
        - 3
        - 5
      - - 1
        - 3
        - 5
      use_weight_norm_in_decoder: true
      posterior_encoder_kernel_size: 3
      posterior_encoder_layers: 8
      posterior_encoder_stacks: 1
      posterior_encoder_base_dilation: 1
      posterior_encoder_dropout_rate: 0.0
      use_weight_norm_in_posterior_encoder: true
      flow_flows: -1
      flow_kernel_size: 5
      flow_base_dilation: 1
      flow_layers: 4
      flow_dropout_rate: 0.0
      use_weight_norm_in_flow: true
      use_only_mean_in_flow: true
      use_phoneme_predictor: false
    discriminator_type: visinger2
    discriminator_params:
      scales: 1
      scale_downsample_pooling: AvgPool1d
      scale_downsample_pooling_params:
        kernel_size: 4
        stride: 2
        padding: 2
      scale_discriminator_params:
        in_channels: 1
        out_channels: 1
        kernel_sizes:
        - 15
        - 41
        - 5
        - 3
        channels: 128
        max_downsample_channels: 1024
        max_groups: 256
        bias: true
        downsample_scales:
        - 4
        - 4
        - 4
        - 4
        nonlinear_activation: LeakyReLU
        nonlinear_activation_params:
          negative_slope: 0.1
        use_weight_norm: true
        use_spectral_norm: false
      follow_official_norm: false
      periods:
      - 2
      - 3
      - 5
      - 7
      - 11
      period_discriminator_params:
        in_channels: 1
        out_channels: 1
        kernel_sizes:
        - 5
        - 3
        channels: 32
        downsample_scales:
        - 3
        - 3
        - 3
        - 3
        - 1
        max_downsample_channels: 1024
        bias: true
        nonlinear_activation: LeakyReLU
        nonlinear_activation_params:
          negative_slope: 0.1
        use_weight_norm: true
        use_spectral_norm: false
      multi_freq_disc_params:
        hop_length_factors:
        - 2.5
        - 5
        - 7.5
        - 10
        - 12.5
        - 15
        hidden_channels:
        - 256
        - 256
        - 256
        - 256
        - 256
        domain: double
        mel_scale: true
        divisors:
        - 32
        - 16
        - 8
        - 4
        - 2
        - 1
        - 1
        strides:
        - 1
        - 2
        - 1
        - 2
        - 1
        - 2
        - 1
    generator_adv_loss_params:
      average_by_discriminators: false
      loss_type: mse
    discriminator_adv_loss_params:
      average_by_discriminators: false
      loss_type: mse
    feat_match_loss_params:
      average_by_discriminators: false
      average_by_layers: false
      include_final_outputs: true
    mel_loss_params:
      fs: 44100
      n_fft: 2048
      hop_length: 512
      win_length: 2048
      window: hann
      n_mels: 80
      fmin: 0
      fmax: 22050
      log_base: null
    lambda_adv: 1.0
    lambda_mel: 45.0
    lambda_feat_match: 2.0
    lambda_dur: 0.1
    lambda_pitch: 10.0
    lambda_phoneme: 1.0
    lambda_kl: 1.0
    sampling_rate: 44100
    cache_generator_outputs: true
optimizer: null
scheduler: null
scheduler_interval: null
scheduler_monitor: null
optimizers:
  generator:
    optimizer:
      _target_: torch.optim.AdamW
      lr: 0.0002
      betas:
      - 0.8
      - 0.99
      eps: 1.0e-09
      weight_decay: 0.0
      _convert_: all
    params: generator
  discriminator:
    optimizer:
      _target_: torch.optim.AdamW
      lr: 0.0002
      betas:
      - 0.8
      - 0.99
      eps: 1.0e-09
      weight_decay: 0.0
      _convert_: all
    params: discriminator
schedulers:
  generator:
    scheduler:
      _target_: torch.optim.lr_scheduler.ExponentialLR
      gamma: 0.998
      _convert_: all
    interval: epoch
  discriminator:
    scheduler:
      _target_: torch.optim.lr_scheduler.ExponentialLR
      gamma: 0.998
      _convert_: all
    interval: epoch
best_model_criterion:
- - valid/generator/loss
  - 5
  - min
seed: 777
generator_first: true
write_collected_feats: true
init: null
parallel:
  env: local
  n_workers: 1
dataloader:
  collate_fn:
    _target_: espnet2.train.collate_fn.CommonCollateFn
    int_pad_value: 0
    float_pad_value: 0.0
    not_sequence:
    - spembs
    - sids
    - lids
    _convert_: all
  train:
    iter_factory:
      _target_: espnet2.iterators.sequence_iter_factory.SequenceIterFactory
      shuffle: true
      num_workers: 4
      collate_fn:
        _target_: espnet2.train.collate_fn.CommonCollateFn
        int_pad_value: 0
        float_pad_value: 0.0
        not_sequence:
        - spembs
        - sids
        - lids
        _convert_: all
      batches:
        type: sorted
        batch_size: 8
        batch_bins: 0
        sort_batch: descending
        shape_files:
        - ./exp/train_visinger2/stats/train/feats_shape
      _convert_: all
  valid:
    iter_factory:
      _target_: espnet2.iterators.sequence_iter_factory.SequenceIterFactory
      shuffle: false
      num_workers: 4
      collate_fn:
        _target_: espnet2.train.collate_fn.CommonCollateFn
        int_pad_value: 0
        float_pad_value: 0.0
        not_sequence:
        - spembs
        - sids
        - lids
        _convert_: all
      batches:
        type: sorted
        batch_size: 8
        batch_bins: 0
        sort_batch: descending
        shape_files:
        - ./exp/train_visinger2/stats/valid/feats_shape
      _convert_: all
trainer:
  accelerator: auto
  devices: 4
  num_nodes: 1
  strategy: ddp_find_unused_parameters_true
  max_epochs: 500
  limit_train_batches: 1000
  check_val_every_n_epoch: 1
  log_every_n_steps: 50
  logger:
  - _target_: lightning.pytorch.loggers.TensorBoardLogger
    save_dir: ./exp/train_visinger2/tensorboard
    name: tb_logger
    _convert_: all
fit:
  ckpt_path: last
create_dataset:
  recipe_dir: .
remove_long_short:
  min_wav_duration: 0.1
  max_wav_duration: 20.0
  splits:
  - train
  - valid
  manifest_paths:
    train: ./data/manifest/train.tsv
    valid: ./data/manifest/valid.tsv
  save_path: ./data/manifest_filtered
create_token_list:
  manifest_path: ./data/manifest_filtered/train.tsv
  save_path: ./data/token_list/phn_none
  filename: tokens.txt
  token_type: phn
  cleaner: null
  g2p: null
  add_symbol:
  - <blank>:0
  - <unk>:1
  - <sos/eos>:-1

Citing ESPnet

@inproceedings{watanabe2018espnet,
  author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and
    Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner
    and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai},
  title={{ESPnet}: End-to-End Speech Processing Toolkit},
  year={2018},
  booktitle={Proceedings of Interspeech},
  pages={2207--2211},
  doi={10.21437/Interspeech.2018-1456}
}