added requirements

adapter.py

@@ -0,0 +1,73 @@
+# --------------------------------------------------------
+# References:
+# https://github.com/jxhe/unify-parameter-efficient-tuning
+# --------------------------------------------------------
+
+import math
+import torch
+import torch.nn as nn
+
+
+class Adapter(nn.Module):
+    def __init__(
+        self,
+        config=None,
+        d_model=768,
+        bottleneck=None,
+        dropout=0.0,
+        init_option="lora",
+        adapter_scalar="1.0",
+        adapter_layernorm_option="none"
+    ):
+        super().__init__()
+        self.n_embd = config.d_model if d_model is None else d_model
+        self.down_size = config.attn_bn if bottleneck is None else bottleneck
+
+        #_before
+        self.adapter_layernorm_option = adapter_layernorm_option
+
+        self.adapter_layer_norm_before = None
+        if adapter_layernorm_option == "in" or adapter_layernorm_option == "out":
+            self.adapter_layer_norm_before = nn.LayerNorm(self.n_embd)
+
+        if adapter_scalar == "learnable_scalar":
+            self.scale = nn.Parameter(torch.ones(1))
+        else:
+            self.scale = float(adapter_scalar)
+
+        self.down_proj = nn.Linear(self.n_embd, self.down_size)
+        self.non_linear_func = nn.ReLU()
+        self.up_proj = nn.Linear(self.down_size, self.n_embd)
+
+        self.dropout = dropout
+        if init_option == "bert":
+            raise NotImplementedError
+        elif init_option == "lora":
+            with torch.no_grad():
+                nn.init.kaiming_uniform_(self.down_proj.weight, a=math.sqrt(5))
+                nn.init.zeros_(self.up_proj.weight)
+                nn.init.zeros_(self.down_proj.bias)
+                nn.init.zeros_(self.up_proj.bias)
+
+    def forward(self, x, add_residual=True, residual=None):
+        residual = x if residual is None else residual
+        if self.adapter_layernorm_option == 'in':
+            x = self.adapter_layer_norm_before(x)
+        
+        down = self.down_proj(x)
+        
+        down = self.non_linear_func(down)
+        down = nn.functional.dropout(down, p=self.dropout, training=self.training)
+        up = self.up_proj(down)
+        
+        up = up * self.scale
+
+        if self.adapter_layernorm_option == 'out':
+            up = self.adapter_layer_norm_before(up)
+
+        if add_residual:
+            output = up + residual
+        else:
+            output = up
+
+        return output

fold_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a05b751ec5c090af650e7aa96278e5d2d77226321be92abb4976103102a2d99
+size 379212283

requirements.txt

@@ -0,0 +1,124 @@
+aif360==0.5.0
+absl-py==1.4.0
+aiohttp==3.8.1
+aiosignal==1.3.1
+antlr4-python3-runtime==4.9.3
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==22.2.0
+audiomentations==0.28.0
+audioread==3.0.0
+blinker==1.5
+Bottleneck==1.3.5
+brotlipy==0.7.0
+cachetools==5.3.0
+certifi==2023.5.7
+cffi==1.15.1
+charset-normalizer==2.0.4
+click==8.1.3
+cloudpickle==2.2.1
+cmake==3.26.4
+contourpy==1.0.7
+cryptography==38.0.4
+cvxopt==1.3.0
+cvxpy==1.3.0
+cycler==0.11.0
+cylp==0.91.5
+decorator==5.1.1
+ecos==2.0.12
+exceptiongroup==1.1.0
+ffmpeg-python==0.2.0
+filelock==3.9.0
+flit_core==3.6.0
+fonttools==4.38.0
+frozenlist==1.3.3
+fsspec==2023.6.0
+future==0.18.3
+google-auth==2.16.0
+google-auth-oauthlib==0.4.6
+grpcio==1.42.0
+holisticai==0.3.0
+huggingface-hub==0.15.1
+HyperPyYAML==1.2.1
+idna==3.4
+importlib-metadata==6.0.0
+importlib-resources==5.10.2
+iniconfig==2.0.0
+joblib==1.2.0
+kiwisolver==1.4.4
+librosa==0.9.2
+lit==16.0.6
+llvmlite==0.39.1
+loralib==0.1.1
+Markdown==3.4.1
+matplotlib==3.7.0
+memory-profiler==0.61.0
+more-itertools==9.1.0
+multidict==6.0.2
+numba==0.56.4
+numexpr==2.8.4
+numpy==1.23.5
+oauthlib==3.2.2
+omegaconf==2.3.0
+osqp==0.6.2.post8
+packaging==22.0
+pandas==1.5.2
+Pillow==9.3.0
+pip==22.3.1
+platformdirs==3.1.1
+pluggy==1.0.0
+pooch==1.6.0
+protobuf==3.15.8
+psutil==5.9.4
+pyasn1==0.4.8
+pyasn1-modules==0.2.7
+pycparser==2.21
+PyJWT==2.6.0
+pyOpenSSL==22.0.0
+pyparsing==3.1.0
+PySocks==1.7.1
+pytest==7.2.1
+python-dateutil==2.8.2
+pytz==2022.7
+pyu2f==0.1.5
+PyYAML==6.0
+qdldl==0.1.5.post3
+regex==2022.7.9
+requests==2.28.1
+requests-oauthlib==1.3.1
+resampy==0.4.2
+rsa==4.9
+ruamel.yaml==0.17.28
+ruamel.yaml.clib==0.2.7
+s3prl==0.4.10
+safetensors==0.3.1  
+scikit-learn==1.2.2
+scipy==1.10.0
+scs==3.2.2
+seaborn==0.12.2
+sentencepiece==0.1.99
+setuptools==59.5.0
+shap==0.41.0
+six==1.16.0
+slicer==0.0.7
+soundfile==0.12.0
+speechbrain==0.5.14
+tempeh==0.1.12
+threadpoolctl==3.1.0
+tiktoken==0.3.1
+tomli==2.0.1
+torch==1.12.1
+torchaudio==0.12.1
+torchvision==0.13.1
+tqdm==4.64.1
+transformers==4.30.2
+triton==2.0.0
+typing_extensions==4.4.0
+urllib3==1.26.14
+Werkzeug==2.1.2
+wheel==0.37.1
+whisper==1.1.10
+yarl==1.7.2
+zipp==3.13.0
+bokeh==2.4.3
+streamlit_bokeh_events

utils.py

@@ -0,0 +1,321 @@
+import json
+import torch
+import random
+import numpy as np
+import transformers
+import argparse, logging
+
+
+transformers.logging.set_verbosity(40)
+
+logging.basicConfig(
+    format='%(asctime)s %(levelname)-3s ==> %(message)s', 
+    level=logging.INFO, 
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+
+def set_seed(seed):
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+    
+def get_results(input_dict):
+    return_dict = dict()
+    return_dict["uar"] = input_dict["uar"]
+    return_dict["acc"] = input_dict["acc"]
+    return_dict["loss"] = input_dict["loss"]
+    return return_dict
+
+def log_epoch_result(
+    result_hist_dict:       dict, 
+    epoch:                  int,
+    train_result:           dict,
+    dev_result:             dict,
+    test_result:            dict,
+    log_dir:                str,
+    fold_idx:               int
+):
+    # read result
+    result_hist_dict[epoch] = dict()
+    result_hist_dict[epoch]["train"] = get_results(train_result)
+    result_hist_dict[epoch]["dev"] = get_results(dev_result)
+    result_hist_dict[epoch]["test"] = get_results(test_result)
+    
+    # dump the dictionary
+    jsonString = json.dumps(result_hist_dict, indent=4)
+    jsonFile = open(str(log_dir.joinpath(f'fold_{fold_idx}.json')), "w")
+    jsonFile.write(jsonString)
+    jsonFile.close()
+    
+    
+def log_best_result(
+    result_hist_dict:       dict, 
+    epoch:                  int,
+    best_dev_uar:           float,
+    best_dev_acc:           float,
+    best_test_uar:          float,
+    best_test_acc:          float,
+    log_dir:                str,
+    fold_idx:               int
+):
+    # log best result
+    result_hist_dict["best"] = dict()
+    result_hist_dict["best"]["dev"], result_hist_dict["best"]["test"] = dict(), dict()
+    result_hist_dict["best"]["dev"]["uar"] = best_dev_uar
+    result_hist_dict["best"]["dev"]["acc"] = best_dev_acc
+    result_hist_dict["best"]["test"]["uar"] = best_test_uar
+    result_hist_dict["best"]["test"]["acc"] = best_test_acc
+
+    # save results for this fold
+    jsonString = json.dumps(result_hist_dict, indent=4)
+    jsonFile = open(str(log_dir.joinpath(f'fold_{fold_idx}.json')), "w")
+    jsonFile.write(jsonString)
+    jsonFile.close()
+
+def parse_finetune_args():
+    # parser
+    parser = argparse.ArgumentParser(description='emo2vec finetune experiments')
+    parser.add_argument(
+        '--data_dir', 
+        default='/media/data/projects/speech-privacy/trust-ser/audio',
+        type=str, 
+        help='raw audio path'
+    )
+
+    parser.add_argument(
+        '--model_dir', 
+        default='/media/data/projects/speech-privacy/trust-ser/model',
+        type=str, 
+        help='model save path'
+    )
+
+    parser.add_argument(
+        '--split_dir', 
+        default='/media/data/projects/speech-privacy/trust-ser/train_split',
+        type=str, 
+        help='train split path'
+    )
+
+    parser.add_argument(
+        '--log_dir', 
+        default='log/finetune',
+        type=str, 
+        help='model save path'
+    )
+
+    parser.add_argument(
+        '--uar_dir', 
+        default='log/uar',
+        type=str, 
+        help='model uar history'
+    )
+
+    parser.add_argument(
+        '--attack_dir', 
+        default='/media/data/projects/speech-privacy/trust-ser/attack',
+        type=str, 
+        help='attack data'
+    )
+    
+    parser.add_argument(
+        '--privacy_attack_dir', 
+        default='/media/data/projects/speech-privacy/trust-ser/privacy',
+        type=str, 
+        help='privacy attack method data'
+    )
+    
+    parser.add_argument(
+        '--privacy_attack', 
+        default='gender',
+        type=str, 
+        help='Privacy attack method'
+    )
+
+    parser.add_argument(
+        '--fairness_dir', 
+        default='/media/data/projects/speech-privacy/trust-ser/fairness',
+        type=str, 
+        help='model save path'
+    )
+
+    parser.add_argument(
+        '--sustainability_dir', 
+        default='/media/data/projects/speech-privacy/trust-ser/sustainability',
+        type=str, 
+        help='model save path'
+    )
+    
+    parser.add_argument(
+        '--attack_method', 
+        default='pgd',
+        type=str, 
+        help='attack method'
+    )
+
+    parser.add_argument(
+        '--pretrain_model', 
+        default='wav2vec2_0',
+        type=str,
+        help="pretrained model type"
+    )
+
+    parser.add_argument(
+        '--finetune', 
+        default='frozen',
+        type=str,
+        help="partial finetune or not"
+    )
+    
+    parser.add_argument(
+        '--learning_rate', 
+        default=0.0002,
+        type=float,
+        help="learning rate",
+    )
+
+    parser.add_argument(
+        '--num_epochs', 
+        default=50,
+        type=int,
+        help="total training rounds",
+    )
+    
+    parser.add_argument(
+        '--optimizer', 
+        default='adam',
+        type=str,
+        help="optimizer",
+    )
+    
+    parser.add_argument(
+        '--dataset',
+        default="iemocap",
+        type=str,
+        help="Dataset name",
+    )
+    
+    parser.add_argument(
+        '--audio_duration', 
+        default=6,
+        type=int,
+        help="audio length for training"
+    )
+
+    parser.add_argument(
+        '--downstream_model', 
+        default='rnn',
+        type=str,
+        help="model type"
+    )
+
+    parser.add_argument(
+        '--num_layers',
+        default=1,
+        type=int,
+        help="num of layers",
+    )
+
+    parser.add_argument(
+        '--snr',
+        default=45,
+        type=int,
+        help="SNR of the audio",
+    )
+
+    parser.add_argument(
+        '--conv_layers',
+        default=3,
+        type=int,
+        help="num of conv layers",
+    )
+
+    parser.add_argument(
+        '--hidden_size',
+        default=256,
+        type=int,
+        help="hidden size",
+    )
+
+    parser.add_argument(
+        '--pooling',
+        default='att',
+        type=str,
+        help="pooling method: att, average",
+    )
+
+    parser.add_argument(
+        '--norm',
+        default='nonorm',
+        type=str,
+        help="normalization or not",
+    )
+    
+    parser.add_argument(
+        '--finetune_method', 
+        default='finetune',
+        type=str, 
+        help='finetune method: adapter, embedding prompt, input prompt'
+    )
+    
+    parser.add_argument(
+        '--adapter_hidden_dim', 
+        default=128,
+        type=int, 
+        help='adapter dimension'
+    )
+    
+    parser.add_argument(
+        '--finetune_emb', 
+        default="all",
+        type=str, 
+        help='adapter dimension'
+    )
+    
+    parser.add_argument(
+        '--embedding_prompt_dim', 
+        default=5,
+        type=int, 
+        help='adapter dimension'
+    )
+    
+    parser.add_argument(
+        '--lora_rank', 
+        default=16,
+        type=int, 
+        help='lora rank'
+    )
+
+    parser.add_argument(
+        '--LPF', 
+        default=False,
+        type=bool, 
+        help='need Low pass filter on Audio'
+    )
+
+    parser.add_argument(
+        '--HPF', 
+        default=False,
+        type=bool, 
+        help='need High pass filter on Audio'
+    )
+    
+    args = parser.parse_args()
+    if args.finetune_method == "adapter" or args.finetune_method == "adapter_l":
+        setting = f'lr{str(args.learning_rate).replace(".", "")}_ep{args.num_epochs}_{args.finetune_method}_{args.adapter_hidden_dim}'
+    elif args.finetune_method == "embedding_prompt":
+        setting = f'lr{str(args.learning_rate).replace(".", "")}_ep{args.num_epochs}_{args.finetune_method}_{args.embedding_prompt_dim}'
+    elif args.finetune_method == "lora":
+        setting = f'lr{str(args.learning_rate).replace(".", "")}_ep{args.num_epochs}_{args.finetune_method}_{args.lora_rank}'
+    elif args.finetune_method == "finetune":
+        setting = f'lr{str(args.learning_rate).replace(".", "")}_ep{args.num_epochs}_{args.finetune_method}'
+    elif args.finetune_method == "combined":
+        setting = f'lr{str(args.learning_rate).replace(".", "")}_ep{args.num_epochs}_{args.finetune_method}_{args.adapter_hidden_dim}_{args.embedding_prompt_dim}_{args.lora_rank}'
+    args.setting = setting
+    if args.finetune_emb != "all":
+        args.setting = args.setting + "_avgtok"
+    
+    return args
+

wavlm_plus.py

@@ -0,0 +1,253 @@
+# part of the code was referenced from SUPERB: https://github.com/s3prl/s3prl
+# and https://github.com/wngh1187/IPET/blob/main/Speechcommands_V2/W2V2/models/W2V2.py
+import os
+import pdb
+import copy
+import torch
+import argparse
+import numpy as np
+import loralib as lora
+import transformers.models.wav2vec2.modeling_wav2vec2 as w2v2
+import transformers.models.wavlm.modeling_wavlm as wavlm
+
+from functools import lru_cache
+from torchaudio.compliance import kaldi
+
+from torch import nn
+from adapter import Adapter
+from collections import OrderedDict
+from typing import Optional, Callable
+from torch.nn import functional as F
+from torch.nn.functional import normalize
+from transformers import  WavLMModel
+
+class WavLMEncoderLayer(nn.Module):
+    def __init__(self, config, has_relative_position_bias: bool = True):
+        super().__init__()
+        self.attention = wavlm.WavLMAttention(
+            embed_dim=config.hidden_size,
+            num_heads=config.num_attention_heads,
+            dropout=config.attention_dropout,
+            num_buckets=config.num_buckets,
+            max_distance=config.max_bucket_distance,
+            has_relative_position_bias=has_relative_position_bias,
+        )
+        self.dropout = nn.Dropout(config.hidden_dropout)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.feed_forward = wavlm.WavLMFeedForward(config)
+        self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.config = config
+        
+        if self.config.finetune_method == "embedding_prompt" or self.config.finetune_method == "combined":
+            self.embed_prompt = nn.Parameter(torch.randn([1, self.config.embedding_prompt_dim, 768]))
+            nn.init.xavier_uniform_(self.embed_prompt)
+        if self.config.finetune_method == "lora" or self.config.finetune_method == "combined":
+            self.feed_forward.intermediate_dense    = lora.Linear(config.hidden_size, config.intermediate_size, r=config.lora_rank)
+            self.feed_forward.output_dense          = lora.Linear(config.intermediate_size, config.hidden_size, r=config.lora_rank)
+            
+        if self.config.finetune_method == "adapter" or self.config.finetune_method == "adapter_l" or self.config.finetune_method == "combined":
+            self.adapter = Adapter(
+                config, 
+                dropout=0.1, 
+                bottleneck=config.adapter_hidden_dim, 
+                adapter_scalar=0.1
+            )
+
+    def forward(self, hidden_states, attention_mask=None, position_bias=None, output_attentions=False, index=0):
+        if self.config.finetune_method == "embedding_prompt" or self.config.finetune_method == "combined":
+            hidden_states = torch.cat((self.embed_prompt.repeat(hidden_states.size(0), 1, 1), hidden_states), dim=1)
+        
+        attn_residual = hidden_states
+        hidden_states, attn_weights, position_bias = self.attention(
+            hidden_states,
+            attention_mask=attention_mask,
+            position_bias=position_bias,
+            output_attentions=output_attentions,
+            index=index,
+        )
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = attn_residual + hidden_states
+        
+        # Adapter
+        if self.config.finetune_method == "adapter":
+            adapt_h = self.adapter(hidden_states)
+
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = hidden_states + self.feed_forward(hidden_states)
+        if self.config.finetune_method == "adapter": 
+            hidden_states = hidden_states + adapt_h
+        if self.config.finetune_method == "adapter_l" or self.config.finetune_method == "combined": 
+            hidden_states = hidden_states + self.adapter(hidden_states)
+        hidden_states = self.final_layer_norm(hidden_states)
+        if self.config.finetune_method == "embedding_prompt" or self.config.finetune_method == "combined":
+            hidden_states = hidden_states[:, self.config.embedding_prompt_dim:, :]
+        outputs = (hidden_states, position_bias)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+   
+class WavLMWrapper(nn.Module):
+    def __init__(
+        self, 
+        args, 
+        hidden_dim=256,
+        output_class_num=7
+    ):
+        super(WavLMWrapper, self).__init__()
+        # 1. We Load the model first with weights
+        self.args = args
+        self.backbone_model = WavLMModel.from_pretrained(
+            "microsoft/wavlm-base-plus",
+            output_hidden_states=True
+        )
+        state_dict = self.backbone_model.state_dict()
+        # 2. Read the model config
+        self.model_config = self.backbone_model.config
+        self.model_config.finetune_method        = args.finetune_method
+        self.model_config.adapter_hidden_dim     = args.adapter_hidden_dim
+        self.model_config.embedding_prompt_dim   = args.embedding_prompt_dim
+        self.model_config.lora_rank              = args.lora_rank
+        
+        # 3. Config encoder layers with adapter or embedding prompt
+        # pdb.set_trace()
+        self.backbone_model.encoder.layers = nn.ModuleList(
+            [WavLMEncoderLayer(self.model_config, has_relative_position_bias=(i == 0)) for i in range(self.model_config.num_hidden_layers)]
+        )
+        # 4. Load the weights back
+        msg = self.backbone_model.load_state_dict(state_dict, strict=False)
+        # 5. Freeze the weights
+        if self.args.finetune_method == "adapter" or self.args.finetune_method == "adapter_l" or self.args.finetune_method == "embedding_prompt" or self.args.finetune_method == "finetune" or self.args.finetune_method == "lora" or self.args.finetune_method == "combined":
+            for name, p in self.backbone_model.named_parameters():
+                if name in msg.missing_keys: p.requires_grad = True
+                else: p.requires_grad = False
+        self.finetune_method = self.args.finetune_method
+        
+        # 6. Downstream models
+        self.model_seq = nn.Sequential(
+            nn.Conv1d(self.model_config.hidden_size, hidden_dim, 1, padding=0),
+            nn.ReLU(),
+            nn.Dropout(p=0.1),
+            nn.Conv1d(hidden_dim, hidden_dim, 1, padding=0),
+            nn.ReLU(),
+            nn.Dropout(p=0.1),
+            nn.Conv1d(hidden_dim, hidden_dim, 1, padding=0)
+        )
+        self.weights = nn.Parameter(torch.zeros(self.model_config.num_hidden_layers))
+        
+        # self.out_layer = nn.Sequential(
+        #     nn.Linear(hidden_dim, hidden_dim),
+        #     nn.ReLU(),
+        #     nn.Linear(hidden_dim, output_class_num),
+        # )
+        self.out_layer = nn.Sequential(
+                nn.Linear(hidden_dim, hidden_dim),
+                nn.ReLU(),
+                nn.Linear(hidden_dim, 2),
+                nn.Sigmoid()
+            )
+        
+    def forward(self, x, length=None):
+        # 1. feature extraction and projections
+        with torch.no_grad():
+            x = self.backbone_model.feature_extractor(x)
+            x = x.transpose(1, 2) # New version of huggingface
+            x, _ = self.backbone_model.feature_projection(x) # New version of huggingface
+        
+        # 2. get length and mask
+        if length is not None:
+            length = self.get_feat_extract_output_lengths(length.detach().cpu())
+            length = length.cuda()
+            
+        # 3. transformer encoding features
+        x = self.backbone_model.encoder(
+            x, output_hidden_states=True
+        ).hidden_states
+        
+        # 4. stacked feature
+        stacked_feature = torch.stack(x, dim=0)[1:]
+        
+        # 5. Weighted sum
+        _, *origin_shape = stacked_feature.shape
+        # Return transformer enc outputs [num_enc_layers, B, T, D]
+        stacked_feature = stacked_feature.view(self.backbone_model.config.num_hidden_layers, -1)
+        norm_weights = F.softmax(self.weights, dim=-1)
+        
+        # Perform weighted average
+        weighted_feature = (norm_weights.unsqueeze(-1) * stacked_feature).sum(dim=0)
+        features = weighted_feature.view(*origin_shape)
+        
+        # 6. Pass the weighted average to point-wise 1D Conv
+        # B x T x D
+        features = features.transpose(1, 2)
+        features = self.model_seq(features)
+        features = features.transpose(1, 2)
+        
+        # 7. Pooling
+        if length is not None:
+            masks = torch.arange(features.size(1)).expand(length.size(0), -1).cuda() < length.unsqueeze(1)
+            masks = masks.float()
+            features = (features * masks.unsqueeze(-1)).sum(1) / length.unsqueeze(1)
+        else:
+            features = torch.mean(features, dim=1)
+        
+        # 8. Output predictions
+        # B x D
+        predicted = self.out_layer(features)
+        return predicted
+        
+    # From huggingface
+    def get_feat_extract_output_lengths(self, input_length):
+        """
+        Computes the output length of the convolutional layers
+        """
+        def _conv_out_length(input_length, kernel_size, stride):
+            # 1D convolutional layer output length formula taken
+            # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html
+            return (input_length - kernel_size) // stride + 1
+        for kernel_size, stride in zip(self.backbone_model.config.conv_kernel, self.backbone_model.config.conv_stride):
+            input_length = _conv_out_length(input_length, kernel_size, stride)
+        return input_length
+
+def prepare_mask(length, shape, dtype):
+    # Modified from huggingface
+    mask = torch.zeros(
+        shape, dtype=dtype
+    )
+    # these two operations makes sure that all values
+    # before the output lengths indices are attended to
+    mask[(torch.arange(mask.shape[0]), length.cpu() - 1)] = 1
+    mask = mask.flip([-1]).cumsum(-1).flip([-1]).bool()
+    return mask
+    
+    
+if __name__ == '__main__':
+    
+    parser = argparse.ArgumentParser(description='emo2vec finetune experiments')
+    parser.add_argument(
+        '--finetune_method', 
+        default='none',
+        type=str, 
+        help='finetune method: adapter, embedding prompt, input prompt'
+    )
+    
+    parser.add_argument(
+        '--adapter_hidden_dim', 
+        default=128,
+        type=int, 
+        help='adapter dimension'
+    )
+    
+    parser.add_argument(
+        '--embedding_prompt_dim', 
+        default=5,
+        type=int, 
+        help='adapter dimension'
+    )
+    
+    args = parser.parse_args()
+    model = WavLMWrapper(args)
+    data = torch.zeros([1, 16000])
+    output = model(data)
+    print(output.shape)