Training in progress, step 100

fine-tune-whisper-streaming.ipynb

@@ -684,36 +684,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 19,
    "id": "0ae3e9af-97b7-4aa0-ae85-20b23b5bcb3a",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "PyTorch: setting up devices\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from transformers import Seq2SeqTrainingArguments\n",
     "\n",
     "training_args = Seq2SeqTrainingArguments(\n",
     "    output_dir=\"./\",\n",
-    "    per_device_train_batch_size=32,\n",
+    "    per_device_train_batch_size=50,\n",
     "    gradient_accumulation_steps=1,  # increase by 2x for every 2x decrease in batch size\n",
     "    learning_rate=1e-5,\n",
     "    warmup_steps=500,\n",
-    "    max_steps=100,   #5000,\n",
+    "    max_steps=3000,   #5000,\n",
     "    gradient_checkpointing=True,\n",
     "    fp16=True,\n",
     "    evaluation_strategy=\"steps\",\n",
-    "    per_device_eval_batch_size=8,\n",
+    "    per_device_eval_batch_size=5,\n",
     "    predict_with_generate=True,\n",
     "    generation_max_length=225,\n",
-    "    save_steps=10,  #100,\n",
-    "    eval_steps=10,\n",
+    "    save_steps=100,  #100,\n",
+    "    eval_steps=100,\n",
     "    logging_steps=1, #25,\n",
     "    report_to=[\"tensorboard\"],\n",
     "    load_best_model_at_end=True,\n",
@@ -742,7 +734,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 20,
    "id": "3ac16b62-b3c0-4c68-8f3d-9ecf471534b2",
    "metadata": {},
    "outputs": [],
@@ -771,7 +763,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 21,
    "id": "d546d7fe-0543-479a-b708-2ebabec19493",
    "metadata": {},
    "outputs": [
@@ -792,7 +784,7 @@
     "    args=training_args,\n",
     "    model=model,\n",
     "    train_dataset=vectorized_datasets[\"train\"],\n",
-    "    eval_dataset=vectorized_datasets[\"test\"].take(64),\n",
+    "    eval_dataset=vectorized_datasets[\"test\"].take(500),\n",
     "    data_collator=data_collator,\n",
     "    compute_metrics=compute_metrics,\n",
     "    tokenizer=processor,\n",
@@ -810,7 +802,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 22,
    "id": "a1ccb9ed-cbc8-4419-91c0-651e9424b672",
    "metadata": {},
    "outputs": [
@@ -867,14 +859,14 @@
       "/home/jovyan/whisper/lib/python3.10/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
       "  warnings.warn(\n",
       "***** Running training *****\n",
-      "  Num examples = 12800\n",
+      "  Num examples = 600000\n",
       "  Num Epochs = 9223372036854775807\n",
-      "  Instantaneous batch size per device = 32\n",
-      "  Total train batch size (w. parallel, distributed & accumulation) = 128\n",
+      "  Instantaneous batch size per device = 50\n",
+      "  Total train batch size (w. parallel, distributed & accumulation) = 200\n",
       "  Gradient Accumulation steps = 1\n",
-      "  Total optimization steps = 100\n",
+      "  Total optimization steps = 3000\n",
       "  Number of trainable parameters = 241734912\n",
-      "Reading metadata...: 18211it [00:02, 8579.93it/s] \n",
+      "Reading metadata...: 18211it [00:02, 8504.35it/s] \n",
       "The following columns in the training set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
       "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
       "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
@@ -886,8 +878,8 @@
        "\n",
        "    <div>\n",
        "      \n",
-       "      <progress value='99' max='100' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
-       "      [ 99/100 31:43 < 00:19, 0.05 it/s, Epoch 0.98/9223372036854775807]\n",
+       "      <progress value='101' max='3000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [ 101/3000 25:24 < 12:24:00, 0.06 it/s, Epoch 0.03/9223372036854775807]\n",
        "    </div>\n",
        "    <table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
@@ -900,58 +892,10 @@
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <td>10</td>\n",
-       "      <td>0.444000</td>\n",
-       "      <td>0.713627</td>\n",
-       "      <td>51.181102</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>20</td>\n",
-       "      <td>0.499800</td>\n",
-       "      <td>0.709538</td>\n",
-       "      <td>55.380577</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>30</td>\n",
-       "      <td>0.432000</td>\n",
-       "      <td>0.702946</td>\n",
-       "      <td>51.443570</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>40</td>\n",
-       "      <td>0.511800</td>\n",
-       "      <td>0.694216</td>\n",
-       "      <td>49.606299</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>50</td>\n",
-       "      <td>0.398100</td>\n",
-       "      <td>0.682893</td>\n",
-       "      <td>48.556430</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>60</td>\n",
-       "      <td>0.473400</td>\n",
-       "      <td>0.669629</td>\n",
-       "      <td>46.719160</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>70</td>\n",
-       "      <td>0.388100</td>\n",
-       "      <td>0.648503</td>\n",
-       "      <td>48.818898</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>80</td>\n",
-       "      <td>0.420200</td>\n",
-       "      <td>0.626310</td>\n",
-       "      <td>50.656168</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>90</td>\n",
-       "      <td>0.342300</td>\n",
-       "      <td>0.602524</td>\n",
-       "      <td>50.918635</td>\n",
+       "      <td>100</td>\n",
+       "      <td>1.069800</td>\n",
+       "      <td>1.207340</td>\n",
+       "      <td>142.617124</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table><p>"
@@ -969,142 +913,20 @@
      "text": [
       "***** Running Evaluation *****\n",
       "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:01, 7728.48it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-10\n",
-      "Configuration saved in ./checkpoint-10/config.json\n",
-      "Model weights saved in ./checkpoint-10/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-10/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-10/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-10/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-10/added_tokens.json\n",
-      "Feature extractor saved in ./preprocessor_config.json\n",
-      "tokenizer config file saved in ./tokenizer_config.json\n",
-      "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:02, 3799.36it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-20\n",
-      "Configuration saved in ./checkpoint-20/config.json\n",
-      "Model weights saved in ./checkpoint-20/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-20/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-20/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-20/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-20/added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:02, 3860.16it/s]\n",
+      "  Batch size = 20\n",
+      "Reading metadata...: 8693it [00:01, 7656.96it/s]\n",
       "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-30\n",
-      "Configuration saved in ./checkpoint-30/config.json\n",
-      "Model weights saved in ./checkpoint-30/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-30/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-30/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-30/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-30/added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:01, 7522.25it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-40\n",
-      "Configuration saved in ./checkpoint-40/config.json\n",
-      "Model weights saved in ./checkpoint-40/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-40/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-40/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-40/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-40/added_tokens.json\n",
+      "Saving model checkpoint to ./checkpoint-100\n",
+      "Configuration saved in ./checkpoint-100/config.json\n",
+      "Model weights saved in ./checkpoint-100/pytorch_model.bin\n",
+      "Feature extractor saved in ./checkpoint-100/preprocessor_config.json\n",
+      "tokenizer config file saved in ./checkpoint-100/tokenizer_config.json\n",
+      "Special tokens file saved in ./checkpoint-100/special_tokens_map.json\n",
+      "added tokens file saved in ./checkpoint-100/added_tokens.json\n",
       "Feature extractor saved in ./preprocessor_config.json\n",
       "tokenizer config file saved in ./tokenizer_config.json\n",
       "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:02, 4089.09it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-50\n",
-      "Configuration saved in ./checkpoint-50/config.json\n",
-      "Model weights saved in ./checkpoint-50/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-50/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-50/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-50/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-50/added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:02, 3824.34it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-60\n",
-      "Configuration saved in ./checkpoint-60/config.json\n",
-      "Model weights saved in ./checkpoint-60/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-60/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-60/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-60/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-60/added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:01, 7724.39it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-70\n",
-      "Configuration saved in ./checkpoint-70/config.json\n",
-      "Model weights saved in ./checkpoint-70/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-70/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-70/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-70/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-70/added_tokens.json\n",
-      "Feature extractor saved in ./preprocessor_config.json\n",
-      "tokenizer config file saved in ./tokenizer_config.json\n",
-      "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:02, 3249.16it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-80\n",
-      "Configuration saved in ./checkpoint-80/config.json\n",
-      "Model weights saved in ./checkpoint-80/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-80/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-80/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-80/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-80/added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 32\n",
-      "Reading metadata...: 8693it [00:02, 3111.55it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-90\n",
-      "Configuration saved in ./checkpoint-90/config.json\n",
-      "Model weights saved in ./checkpoint-90/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-90/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-90/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-90/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-90/added_tokens.json\n",
-      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
-      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
+      "added tokens file saved in ./added_tokens.json\n"
      ]
     }
    ],

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9137e7fd994d7f2213b941c29befb2154671196a10167f3eca8be6be69499fb3
+oid sha256:c39156e0baeaa2a0bbfb0619976fb64003fc312ae1fa5a05b6b98b78c47fa454
 size 967099139

runs/Dec06_02-49-42_04812eda968b/events.out.tfevents.1670295010.04812eda968b.1031606.3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a603818d95c33f5e84e4a76a735bd22f4bf244d641e2ff5d8b1c5b7f9f67e2f8
-size 22710
+oid sha256:1a0738a5ca78b0a1c5521ecdaa8f89a160e520d861e79d9fdf3d6dad00e17eed
+size 23058

runs/Dec06_03-53-43_04812eda968b/1670298849.4736288/events.out.tfevents.1670298849.04812eda968b.1031606.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:040321fe8626593fba40aa3fe2d1189172b567f9b9aa5c2d31edbbfe5cf21f6b
+size 5701

runs/Dec06_03-53-43_04812eda968b/events.out.tfevents.1670298849.04812eda968b.1031606.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0254f34c208105396800997d02b76e1a43ec2f6420e112c391bb30cb4cdb44ce
+size 5580

runs/Dec06_03-57-56_04812eda968b/1670299090.1920335/events.out.tfevents.1670299090.04812eda968b.1031606.8

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf3889b8a853fef26f36df5d508da50972ac19277696c5fac25525c05032bb5d
+size 5701

runs/Dec06_03-57-56_04812eda968b/events.out.tfevents.1670299090.04812eda968b.1031606.7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fd9e1f17d3a11a984e8cc89e14fd25e9b304fff02ca7d0a91f4472462294bb1
+size 19594

runs/Dec06_06-58-23_04812eda968b/1670309913.1115/events.out.tfevents.1670309913.04812eda968b.2449868.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f2e0f225220c341b5870d143fa4387b7094084eb8ec22a18d8836f017cbaad4
+size 5701

runs/Dec06_06-58-23_04812eda968b/events.out.tfevents.1670309913.04812eda968b.2449868.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f895155f75dec6d2940f49f5c2da89db62ffa5ff6260646f4a8e92b6fe180ef7
+size 19594

runs/Dec06_07-40-22_04812eda968b/1670312431.563092/events.out.tfevents.1670312431.04812eda968b.2650899.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e158bd332d51cec6795201a51029faadec007da2bee3c4f4e3562dc287a3fb17
+size 5701

runs/Dec06_07-40-22_04812eda968b/events.out.tfevents.1670312431.04812eda968b.2650899.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9133eed6ce04b45b52ff52dc5a94d9f934513443190fe9cb6f4fecd2ae1e64d
+size 19905

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc766c5602f1abe460fe642854fc15c8de61a50b681d12e01c00c241956762e5
+oid sha256:1ac204e2762aaab2d3f59ce61d73b8a7fb5e1f9724f92d59b71383f7a17e163f
 size 3503