Training in progress, step 200

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1061349853637e78beb25546e7229b776288b8bc0b1e9ceedb20bfff093efd7
+oid sha256:574b6173d421e48618cab813ba4c01980928c611f41fd03eba8de73360a0aa8a
 size 4988522632

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ecf70e6c84ac3ab87aa49dc18fda2bc0d34397121771d9a136472128766a909
+oid sha256:bdd6d69ff075939395f9774225b8131baefa33d9b34ec513bd136f2517e28f69
 size 1118042580

train_conv_slurm_full.py

@@ -17,16 +17,17 @@ import numpy as np
 # ========================
 WANDB_PROJECT_NAME = "mmlm-conv-full"
 WANDB_API_KEY = "0793be66347fa388f401f66cb39fd661452d660d"
+# DATASET = load_dataset("voidful/all_conv_data")['train']
 DATASET = load_dataset("voidful/all_conv_data_filtered_small")['train']
 # DATASET = datasets.load_from_disk("/mnt/home/ntuspeechlabtaipei1/anthony/Soundon-TTS-preprocessing/hf_dialogue_chinese_llama31_70B_user_long_2_with_silence")
 LM_MODEL_NAME = "voidful/Llama-3.2-8B-Whisper"
 OUTPUT_DIR = "/mnt/home/ntuspeechlabtaipei1/mmlm-conv-training-full"
 MODEL_SAVE_PATH = "/mnt/home/ntuspeechlabtaipei1/mmlm-conv-model-full"
 TRAIN_TEST_SPLIT_RATIO = 0.1
-EPOCHS = 300
+EPOCHS = 5000
 BATCH_SIZE = 1
-LEARNING_RATE = 8e-4
-GRADIENT_ACCUMULATION_STEPS = 2
+LEARNING_RATE = 1e-6
+GRADIENT_ACCUMULATION_STEPS = 20
 USE_BF16 = True
 USE_FP16 = False
 LOGGING_STEPS = 1
@@ -53,6 +54,7 @@ def initialize_wandb():
         group="mmlm",
     )
 
+
 class CustomDataset(Dataset):
     """Custom dataset class for handling audio-text data."""
 
@@ -67,6 +69,7 @@ class CustomDataset(Dataset):
         entry = self.data
         # print(len(entry[idx]["user_audio_path"]['array']),entry[idx]["user_audio_path"]['array'])
         audio_path = torch.tensor(entry[idx]["user_audio_path"]['array'])
+        # audio_path = entry[idx]["user_audio_path"]
         # if not os.path.exists(audio_path):
         #     audio_path = os.path.join("/mnt/home/ntuspeechlabtaipei1/anthony/Soundon-TTS-preprocessing/", audio_path)
         audio_tensor = load_audio_to_tensor(audio_path)[0]
@@ -76,8 +79,8 @@ class CustomDataset(Dataset):
         user_text_with_pad = text_with_pad[0]
         user_text_with_pad = "[PAD]" + user_text_with_pad
         audio_tensor = torch.cat([audio_tensor[0], torch.zeros(int(24000 * 0.08 * 1))], dim=0).unsqueeze(dim=0)
-        # machine_text_with_pad = text_with_pad[1]
-        machine_text_with_pad = text_with_pad[1][5:] + "[PAD]"
+        machine_text_with_pad = text_with_pad[1]
+        # machine_text_with_pad = text_with_pad[1][5:] + "[PAD]"
         audio_unit = np.array(entry[idx]["machine_unit"])
 
         zero_sequences = []  # To store start and end times
@@ -127,7 +130,6 @@ class CustomDataset(Dataset):
         }
 
 
-
 class CustomDataCollator:
     """Custom data collator for batching audio and text inputs."""
 
@@ -174,8 +176,8 @@ def main():
     # Split dataset
     # data = data.train_test_split(test_size=0.5, seed=42)
     data = data.shuffle(seed=42)
-    subset_size = 100
-    data = data.select(range(subset_size))
+    # subset_size = 4000
+    # data = data.select(range(subset_size))
     train_dataset = CustomDataset(data, tokenizer)
     # eval_dataset = CustomDataset(data['test'], tokenizer)
     # train_dataset = CustomDataset(data.select([0, 1, 2, 3, 4]), tokenizer)
@@ -226,8 +228,8 @@ def main():
     )
 
     # Train and evaluate model
-    # resume_from_checkpoint = '/mnt/home/ntuspeechlabtaipei1/mmlm-conv-training-fixed-10k/checkpoint-2000/'
-    trainer.train()
+    # resume_from_checkpoint = ‘/mnt/home/ntuspeechlabtaipei1/mmlm-conv-training-full/checkpoint-75200/’
+    trainer.train(resume_from_checkpoint='/mnt/home/ntuspeechlabtaipei1/mmlm-conv-training-full/checkpoint-75200/')
 
     # Save model
     trainer.save_model(MODEL_SAVE_PATH)

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:043756452dd25ffceba12b66887f0f266cae4544bb031d32637bfbae7b033734
+oid sha256:c244fb7f19cf364a14a14281a7e64bd1dad296cfa501f160bb8430cbc9730013
 size 7672