Update training.ipynb to v0.2.0

training.ipynb

@@ -109,6 +109,8 @@
         "            self.data = [row for row in reader]\n",
         "\n",
         "    def __getitem__(self, index: int) -> Dict[str, torch.Tensor]:\n",
+        "#         with open('train_log.txt', 'a+') as log_file:\n",
+        "#             log_file.write(f'reading data[{index}] {self.data[index]}\\n')\n",
         "        src, trg = self.data[index]\n",
         "        embeddings = self.src_tokenizer(src, return_attention_mask=False, return_token_type_ids=False)\n",
         "        embeddings['labels'] = self.trg_tokenizer.build_inputs_with_special_tokens(self.trg_tokenizer(trg, return_attention_mask=False)['input_ids'])\n",
@@ -121,13 +123,26 @@
         "DATA_ROOT = './output'\n",
         "FILE_FFAC_FULL = 'ffac_full.csv'\n",
         "FILE_FFAC_TEST = 'ffac_test.csv'\n",
-        "# FILE_JA_KO_TRAIN = 'ja_ko_train.csv'\n",
-        "# FILE_JA_KO_TEST = 'ja_ko_test.csv'\n",
+        "FILE_JA_KO_TRAIN = 'ja_ko_train.csv'\n",
+        "FILE_JA_KO_TEST = 'ja_ko_test.csv'\n",
         "\n",
-        "train_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_FFAC_FULL}')\n",
-        "eval_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_FFAC_TEST}') \n",
-        "# train_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_JA_KO_TRAIN}')\n",
-        "# eval_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_JA_KO_TEST}')        "
+        "# train_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_FFAC_FULL}')\n",
+        "# eval_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_FFAC_TEST}') \n",
+        "train_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_JA_KO_TRAIN}')\n",
+        "eval_dataset = PairedDataset(src_tokenizer, trg_tokenizer, f'{DATA_ROOT}/{FILE_JA_KO_TEST}')        "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# be sure to check the column count of each dataset if you encounter \"ValueError: too many values to unpack (expected 2)\"\n",
+        "# at the `src, trg = self.data[index]`\n",
+        "# The `cat ffac_full.csv tteb_train.csv > ja_ko_train.csv` command may be the reason.\n",
+        "# the last row of first csv and first row of second csv is merged and that's why 3rd column is created (which arouse ValueError)\n",
+        "# debug_data = train_dataset.data\n"
       ]
     },
     {
@@ -175,12 +190,12 @@
         "    do_eval=True,\n",
         "    evaluation_strategy=\"epoch\",\n",
         "    save_strategy=\"epoch\",\n",
-        "#     num_train_epochs=5,\n",
-        "    num_train_epochs=25,\n",
-        "#     per_device_train_batch_size=32,\n",
-        "    per_device_train_batch_size=64,\n",
-        "#     per_device_eval_batch_size=32,\n",
-        "    per_device_eval_batch_size=64,\n",
+        "    num_train_epochs=3,\n",
+        "    # num_train_epochs=25,\n",
+        "    per_device_train_batch_size=30,\n",
+        "    # per_device_train_batch_size=64,\n",
+        "    per_device_eval_batch_size=30,\n",
+        "    # per_device_eval_batch_size=64,\n",
         "    warmup_ratio=0.1,\n",
         "    gradient_accumulation_steps=4,\n",
         "    save_total_limit=5,\n",
@@ -229,7 +244,9 @@
       "source": [
         "trainer.train()\n",
         "\n",
-        "model.save_pretrained(\"dump/best_model\")"
+        "model.save_pretrained(\"dump/best_model\")\n",
+        "src_tokenizer.save_pretrained(\"dump/best_model/src_tokenizer\")\n",
+        "trg_tokenizer.save_pretrained(\"dump/best_model/trg_tokenizer\")"
       ]
     }
   ],