model improved

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 30.0,
-    "eval_loss": 0.03293671831488609,
-    "eval_mem_cpu_alloc_delta": 85377,
-    "eval_mem_cpu_peaked_delta": 33067,
+    "eval_loss": 0.027720022946596146,
+    "eval_mem_cpu_alloc_delta": 0,
+    "eval_mem_cpu_peaked_delta": 0,
     "eval_mem_gpu_alloc_delta": 0,
     "eval_mem_gpu_peaked_delta": 569275904,
-    "eval_runtime": 4.0268,
-    "eval_samples": 23,
-    "eval_samples_per_second": 5.712,
-    "init_mem_cpu_alloc_delta": 1868923,
-    "init_mem_cpu_peaked_delta": 18306,
+    "eval_runtime": 8.6733,
+    "eval_samples": 49,
+    "eval_samples_per_second": 5.65,
+    "init_mem_cpu_alloc_delta": 702398464,
+    "init_mem_cpu_peaked_delta": 0,
     "init_mem_gpu_alloc_delta": 1370738688,
     "init_mem_gpu_peaked_delta": 0,
-    "perplexity": 1.0334851364909519,
-    "train_mem_cpu_alloc_delta": 480835,
-    "train_mem_cpu_peaked_delta": 329934,
+    "perplexity": 1.0281077975307098,
+    "train_mem_cpu_alloc_delta": 18444288,
+    "train_mem_cpu_peaked_delta": 0,
     "train_mem_gpu_alloc_delta": 4036706816,
     "train_mem_gpu_peaked_delta": 8663361536,
-    "train_runtime": 399.4076,
-    "train_samples": 23,
-    "train_samples_per_second": 1.728
+    "train_runtime": 924.1697,
+    "train_samples": 49,
+    "train_samples_per_second": 1.591
 }

config.json

@@ -1,14 +1,16 @@
 {
   "_name_or_path": "rinna/japanese-gpt2-medium",
   "activation_function": "gelu_new",
-  "architectures": ["GPT2LMHeadModel"],
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
   "attn_pdrop": 0.1,
   "bos_token_id": 1,
   "embd_pdrop": 0.1,
   "eos_token_id": 2,
   "gradient_checkpointing": false,
   "initializer_range": 0.02,
-  "layer_norm_epsilon": 1e-5,
+  "layer_norm_epsilon": 1e-05,
   "model_type": "gpt2",
   "n_ctx": 1024,
   "n_embd": 1024,
@@ -26,11 +28,10 @@
   "task_specific_params": {
     "text-generation": {
       "do_sample": true,
-      "max_length": 500,
-      "min_length": 200
+      "max_length": 50
     }
   },
-  "transformers_version": "4.4.2",
+  "transformers_version": "4.6.1",
   "use_cache": true,
   "vocab_size": 32000
 }

eval_results.json

@@ -1,12 +1,12 @@
 {
     "epoch": 30.0,
-    "eval_loss": 0.03293671831488609,
-    "eval_mem_cpu_alloc_delta": 85377,
-    "eval_mem_cpu_peaked_delta": 33067,
+    "eval_loss": 0.027720022946596146,
+    "eval_mem_cpu_alloc_delta": 0,
+    "eval_mem_cpu_peaked_delta": 0,
     "eval_mem_gpu_alloc_delta": 0,
     "eval_mem_gpu_peaked_delta": 569275904,
-    "eval_runtime": 4.0268,
-    "eval_samples": 23,
-    "eval_samples_per_second": 5.712,
-    "perplexity": 1.0334851364909519
+    "eval_runtime": 8.6733,
+    "eval_samples": 49,
+    "eval_samples_per_second": 5.65,
+    "perplexity": 1.0281077975307098
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d12438c9ecb3f0be9106cde6cc82fb132611c13e2535515f2ad5b8285a9a3662
+oid sha256:1bda60b0d6cc78a1cdb2b1e3edf663f5e76a0d66dca1cffb8b63ee79dbb26374
 size 1369800665

train_results.json

@@ -1,14 +1,14 @@
 {
     "epoch": 30.0,
-    "init_mem_cpu_alloc_delta": 1868923,
-    "init_mem_cpu_peaked_delta": 18306,
+    "init_mem_cpu_alloc_delta": 702398464,
+    "init_mem_cpu_peaked_delta": 0,
     "init_mem_gpu_alloc_delta": 1370738688,
     "init_mem_gpu_peaked_delta": 0,
-    "train_mem_cpu_alloc_delta": 480835,
-    "train_mem_cpu_peaked_delta": 329934,
+    "train_mem_cpu_alloc_delta": 18444288,
+    "train_mem_cpu_peaked_delta": 0,
     "train_mem_gpu_alloc_delta": 4036706816,
     "train_mem_gpu_peaked_delta": 8663361536,
-    "train_runtime": 399.4076,
-    "train_samples": 23,
-    "train_samples_per_second": 1.728
+    "train_runtime": 924.1697,
+    "train_samples": 49,
+    "train_samples_per_second": 1.591
 }

trainer_state.json

@@ -2,28 +2,34 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 30.0,
-  "global_step": 690,
+  "global_step": 1470,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 21.74,
-      "learning_rate": 1.3768115942028985e-05,
-      "loss": 1.1133,
+      "epoch": 10.2,
+      "learning_rate": 3.2993197278911564e-05,
+      "loss": 1.9694,
       "step": 500
     },
+    {
+      "epoch": 20.41,
+      "learning_rate": 1.5986394557823133e-05,
+      "loss": 0.4335,
+      "step": 1000
+    },
     {
       "epoch": 30.0,
-      "step": 690,
-      "total_flos": 1424967598080000.0,
-      "train_runtime": 399.4076,
-      "train_samples_per_second": 1.728
+      "step": 1470,
+      "total_flos": 3035800535040000.0,
+      "train_runtime": 924.1697,
+      "train_samples_per_second": 1.591
     }
   ],
-  "max_steps": 690,
+  "max_steps": 1470,
   "num_train_epochs": 30,
-  "total_flos": 1424967598080000.0,
+  "total_flos": 3035800535040000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59cb39dc91671c522f484e1862e34286b78345983173bb19730f50a974ad81f2
-size 2287
+oid sha256:bf9b36f634f2e25ce41f64466f8899bfd197b34e1f4605d3c6f8209bec172180
+size 2415