manual commit

config.json

@@ -6,7 +6,7 @@
   "add_adapter": false,
   "apply_spec_augment": true,
   "architectures": [
-    "Wav2Vec2ForCTC"
+    "Wav2Vec2ForPreTraining"
   ],
   "attention_dropout": 0.0,
   "bos_token_id": 1,

run.sh

@@ -13,16 +13,16 @@ python run_speech_recognition_ctc_bnb.py \
         --output_dir="./" \
         --overwrite_output_dir \
         --num_train_epochs="20" \
-        --per_device_train_batch_size="8" \
-        --per_device_eval_batch_size="8" \
+        --per_device_train_batch_size="16" \
+        --per_device_eval_batch_size="16" \
         --gradient_accumulation_steps="4" \
         --learning_rate="7.5e-5" \
         --warmup_steps="100" \
         --evaluation_strategy="steps" \
 	--length_column_name="input_length" \
         --chars_to_ignore , ? . ! \- \; \: \" " % ‘ " � — ’ … – \
-        --save_steps="100" \
-        --eval_steps="100" \
+        --save_steps="250" \
+        --eval_steps="250" \
         --logging_steps="100" \
         --layerdrop="0.0" \
         --activation_dropout="0.1" \

run_speech_recognition_ctc_bnb.py

@@ -686,25 +686,25 @@ def main():
     # Instantiate custom data collator
     data_collator = DataCollatorCTCWithPadding(processor=processor)
 
-    decay_parameters = get_parameter_names(model, [torch.nn.LayerNorm])
-    decay_parameters = [name for name in decay_parameters if "bias" not in name]
-    optimizer_grouped_parameters = [
-        {
-            "params": [p for n, p in model.named_parameters() if n in decay_parameters],
-            "weight_decay": training_args.weight_decay,
-        },
-        {
-            "params": [p for n, p in model.named_parameters() if n not in decay_parameters],
-            "weight_decay": 0.0,
-        },
-    ]
-    optimizer = bnb.optim.Adam8bit(
-        params=optimizer_grouped_parameters,
-        betas=(training_args.adam_beta1, training_args.adam_beta2),
-        eps=training_args.adam_epsilon,
-    )
-
-    optimizers = (optimizer, None)
+    #decay_parameters = get_parameter_names(model, [torch.nn.LayerNorm])
+    #decay_parameters = [name for name in decay_parameters if "bias" not in name]
+    #optimizer_grouped_parameters = [
+    #    {
+    #        "params": [p for n, p in model.named_parameters() if n in decay_parameters],
+    #        "weight_decay": training_args.weight_decay,
+    #    },
+    #    {
+    #        "params": [p for n, p in model.named_parameters() if n not in decay_parameters],
+    #        "weight_decay": 0.0,
+    #    },
+    #]
+    #optimizer = bnb.optim.Adam8bit(
+    #    params=optimizer_grouped_parameters,
+    #    betas=(training_args.adam_beta1, training_args.adam_beta2),
+    #    eps=training_args.adam_epsilon,
+    #)
+
+    #optimizers = (optimizer, None)
 
     # Initialize Trainer
     trainer = Trainer(
@@ -715,7 +715,7 @@ def main():
         train_dataset=vectorized_datasets["train"] if training_args.do_train else None,
         eval_dataset=vectorized_datasets["eval"] if training_args.do_eval else None,
         tokenizer=feature_extractor,
-        optimizers=optimizers,
+        #optimizers=optimizers,
     )
 
     # 8. Finally, we can start training

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}