:seedling: add baseline script

added_tokens.json

@@ -1 +0,0 @@
-{"<s>": 47, "</s>": 48}

all_results.json

@@ -1,14 +0,0 @@
-{
-    "epoch": 0.01,
-    "eval_loss": 5.129676342010498,
-    "eval_runtime": 137.0925,
-    "eval_samples": 2609,
-    "eval_samples_per_second": 19.031,
-    "eval_steps_per_second": 1.196,
-    "eval_wer": 1.0,
-    "train_loss": 6.290350723266601,
-    "train_runtime": 127.1004,
-    "train_samples": 5705,
-    "train_samples_per_second": 0.315,
-    "train_steps_per_second": 0.079
-}

config.json

@@ -1,107 +0,0 @@
-{
-  "_name_or_path": "facebook/wav2vec2-xls-r-300m",
-  "activation_dropout": 0.0,
-  "adapter_kernel_size": 3,
-  "adapter_stride": 2,
-  "add_adapter": false,
-  "apply_spec_augment": true,
-  "architectures": [
-    "Wav2Vec2ForCTC"
-  ],
-  "attention_dropout": 0.0,
-  "bos_token_id": 1,
-  "classifier_proj_size": 256,
-  "codevector_dim": 768,
-  "contrastive_logits_temperature": 0.1,
-  "conv_bias": true,
-  "conv_dim": [
-    512,
-    512,
-    512,
-    512,
-    512,
-    512,
-    512
-  ],
-  "conv_kernel": [
-    10,
-    3,
-    3,
-    3,
-    3,
-    2,
-    2
-  ],
-  "conv_stride": [
-    5,
-    2,
-    2,
-    2,
-    2,
-    2,
-    2
-  ],
-  "ctc_loss_reduction": "mean",
-  "ctc_zero_infinity": false,
-  "diversity_loss_weight": 0.1,
-  "do_stable_layer_norm": true,
-  "eos_token_id": 2,
-  "feat_extract_activation": "gelu",
-  "feat_extract_dropout": 0.0,
-  "feat_extract_norm": "layer",
-  "feat_proj_dropout": 0.0,
-  "feat_quantizer_dropout": 0.0,
-  "final_dropout": 0.0,
-  "hidden_act": "gelu",
-  "hidden_dropout": 0.0,
-  "hidden_size": 1024,
-  "initializer_range": 0.02,
-  "intermediate_size": 4096,
-  "layer_norm_eps": 1e-05,
-  "layerdrop": 0.0,
-  "mask_feature_length": 10,
-  "mask_feature_min_masks": 0,
-  "mask_feature_prob": 0.0,
-  "mask_time_length": 10,
-  "mask_time_min_masks": 2,
-  "mask_time_prob": 0.05,
-  "model_type": "wav2vec2",
-  "num_adapter_layers": 3,
-  "num_attention_heads": 16,
-  "num_codevector_groups": 2,
-  "num_codevectors_per_group": 320,
-  "num_conv_pos_embedding_groups": 16,
-  "num_conv_pos_embeddings": 128,
-  "num_feat_extract_layers": 7,
-  "num_hidden_layers": 24,
-  "num_negatives": 100,
-  "output_hidden_size": 1024,
-  "pad_token_id": 46,
-  "proj_codevector_dim": 768,
-  "tdnn_dilation": [
-    1,
-    2,
-    3,
-    1,
-    1
-  ],
-  "tdnn_dim": [
-    512,
-    512,
-    512,
-    512,
-    1500
-  ],
-  "tdnn_kernel": [
-    5,
-    3,
-    3,
-    1,
-    1
-  ],
-  "torch_dtype": "float32",
-  "transformers_version": "4.16.0.dev0",
-  "use_weighted_layer_sum": false,
-  "vocab_size": 49,
-  "xvector_output_dim": 512
-}

eval_results.json

@@ -1,9 +0,0 @@
-{
-    "epoch": 0.01,
-    "eval_loss": 5.129676342010498,
-    "eval_runtime": 137.0925,
-    "eval_samples": 2609,
-    "eval_samples_per_second": 19.031,
-    "eval_steps_per_second": 1.196,
-    "eval_wer": 1.0
-}

preprocessor_config.json

@@ -1,9 +0,0 @@
-{
-  "do_normalize": true,
-  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
-  "feature_size": 1,
-  "padding_side": "right",
-  "padding_value": 0,
-  "return_attention_mask": true,
-  "sampling_rate": 16000
-}

pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c79d9bfd900ed445b471b21027361e204c7b99e85ed9307a769eb70452db5048
-size 1262124593

run_speech_recognition_ctc.py → run_speech_recognition_ctc_bnb.py

@@ -30,6 +30,7 @@ import numpy as np
 import torch
 from datasets import DatasetDict, load_dataset, load_metric
 
+import bitsandbytes as bnb
 import transformers
 from transformers import (
     AutoConfig,
@@ -43,6 +44,7 @@ from transformers import (
     Wav2Vec2Processor,
     set_seed,
 )
+from transformers.trainer_pt_utils import get_parameter_names
 from transformers.trainer_utils import get_last_checkpoint, is_main_process
 from transformers.utils import check_min_version
 from transformers.utils.versions import require_version
@@ -661,6 +663,26 @@ def main():
     # Instantiate custom data collator
     data_collator = DataCollatorCTCWithPadding(processor=processor)
 
+    decay_parameters = get_parameter_names(model, [torch.nn.LayerNorm])
+    decay_parameters = [name for name in decay_parameters if "bias" not in name]
+    optimizer_grouped_parameters = [
+        {
+            "params": [p for n, p in model.named_parameters() if n in decay_parameters],
+            "weight_decay": training_args.weight_decay,
+        },
+        {
+            "params": [p for n, p in model.named_parameters() if n not in decay_parameters],
+            "weight_decay": 0.0,
+        },
+    ]
+    optimizer = bnb.optim.Adam8bit(
+        params=optimizer_grouped_parameters,
+        betas=(training_args.adam_beta1, training_args.adam_beta2),
+        eps=training_args.adam_epsilon,
+    )
+
+    optimizers = (optimizer, None)
+
     # Initialize Trainer
     trainer = Trainer(
         model=model,
@@ -670,6 +692,7 @@ def main():
         train_dataset=vectorized_datasets["train"] if training_args.do_train else None,
         eval_dataset=vectorized_datasets["eval"] if training_args.do_eval else None,
         tokenizer=feature_extractor,
+        optimizers=optimizers,
     )
 
     # 8. Finally, we can start training

special_tokens_map.json

@@ -1 +0,0 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -1 +0,0 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

train_results.json

@@ -1,8 +0,0 @@
-{
-    "epoch": 0.01,
-    "train_loss": 6.290350723266601,
-    "train_runtime": 127.1004,
-    "train_samples": 5705,
-    "train_samples_per_second": 0.315,
-    "train_steps_per_second": 0.079
-}

trainer_state.json

@@ -1,25 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 0.00700770847932726,
-  "global_step": 10,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.01,
-      "step": 10,
-      "total_flos": 1.12376657426688e+16,
-      "train_loss": 6.290350723266601,
-      "train_runtime": 127.1004,
-      "train_samples_per_second": 0.315,
-      "train_steps_per_second": 0.079
-    }
-  ],
-  "max_steps": 10,
-  "num_train_epochs": 1,
-  "total_flos": 1.12376657426688e+16,
-  "trial_name": null,
-  "trial_params": null
-}

training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c65283301b8c8bb1ae36123e130388a1d7315758fdd0ecf252815e5209075192
-size 2991

vocab.json

@@ -1 +0,0 @@
-{"!": 1, "\"": 2, "'": 3, ",": 4, "-": 5, ".": 6, ";": 7, "?": 8, "a": 9, "b": 10, "c": 11, "d": 12, "e": 13, "f": 14, "g": 15, "h": 16, "i": 17, "j": 18, "k": 19, "l": 20, "m": 21, "n": 22, "o": 23, "p": 24, "q": 25, "r": 26, "s": 27, "t": 28, "u": 29, "v": 30, "w": 31, "x": 32, "y": 33, "z": 34, "ä": 35, "õ": 36, "ö": 37, "ü": 38, "š": 39, "ž": 40, "̇": 41, "’": 42, "“": 43, "„": 44, "|": 0, "[UNK]": 45, "[PAD]": 46}