first commit

config.json

@@ -0,0 +1,41 @@
+{
+  "_num_labels": 1,
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 1280,
+  "n_head": 20,
+  "n_inner": null,
+  "n_layer": 36,
+  "n_positions": 1024,
+  "resid_pdrop": 0.1,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "total_flos": 418496391413760000,
+  "vocab_size": 50257
+}

log_history.json

@@ -0,0 +1,156 @@
+[
+  {
+    "loss": 3.608123046875,
+    "learning_rate": 4.777024616482341e-05,
+    "epoch": 0.04459159671359932,
+    "total_flos": 19022563246080000,
+    "step": 500
+  },
+  {
+    "loss": 3.56192529296875,
+    "learning_rate": 4.554049232964681e-05,
+    "epoch": 0.08918319342719865,
+    "total_flos": 38045126492160000,
+    "step": 1000
+  },
+  {
+    "loss": 3.54396728515625,
+    "learning_rate": 4.331073849447021e-05,
+    "epoch": 0.13377479014079796,
+    "total_flos": 57067689738240000,
+    "step": 1500
+  },
+  {
+    "loss": 3.5339208984375,
+    "learning_rate": 4.1080984659293615e-05,
+    "epoch": 0.1783663868543973,
+    "total_flos": 76090252984320000,
+    "step": 2000
+  },
+  {
+    "loss": 3.5219326171875,
+    "learning_rate": 3.885123082411702e-05,
+    "epoch": 0.22295798356799662,
+    "total_flos": 95112816230400000,
+    "step": 2500
+  },
+  {
+    "loss": 3.508724609375,
+    "learning_rate": 3.6621476988940425e-05,
+    "epoch": 0.2675495802815959,
+    "total_flos": 114135379476480000,
+    "step": 3000
+  },
+  {
+    "loss": 3.505310546875,
+    "learning_rate": 3.439172315376383e-05,
+    "epoch": 0.3121411769951953,
+    "total_flos": 133157942722560000,
+    "step": 3500
+  },
+  {
+    "loss": 3.497248046875,
+    "learning_rate": 3.2161969318587235e-05,
+    "epoch": 0.3567327737087946,
+    "total_flos": 152180505968640000,
+    "step": 4000
+  },
+  {
+    "loss": 3.48703125,
+    "learning_rate": 2.993221548341063e-05,
+    "epoch": 0.4013243704223939,
+    "total_flos": 171203069214720000,
+    "step": 4500
+  },
+  {
+    "loss": 3.48241796875,
+    "learning_rate": 2.7702461648234034e-05,
+    "epoch": 0.44591596713599324,
+    "total_flos": 190225632460800000,
+    "step": 5000
+  },
+  {
+    "loss": 3.4734921875,
+    "learning_rate": 2.547270781305744e-05,
+    "epoch": 0.49050756384959254,
+    "total_flos": 209248195706880000,
+    "step": 5500
+  },
+  {
+    "loss": 3.47524609375,
+    "learning_rate": 2.3242953977880844e-05,
+    "epoch": 0.5350991605631918,
+    "total_flos": 228270758952960000,
+    "step": 6000
+  },
+  {
+    "loss": 3.4679453125,
+    "learning_rate": 2.1013200142704246e-05,
+    "epoch": 0.5796907572767912,
+    "total_flos": 247293322199040000,
+    "step": 6500
+  },
+  {
+    "loss": 3.45712890625,
+    "learning_rate": 1.878344630752765e-05,
+    "epoch": 0.6242823539903906,
+    "total_flos": 266315885445120000,
+    "step": 7000
+  },
+  {
+    "loss": 3.458171875,
+    "learning_rate": 1.6553692472351056e-05,
+    "epoch": 0.6688739507039898,
+    "total_flos": 285338448691200000,
+    "step": 7500
+  },
+  {
+    "loss": 3.45421875,
+    "learning_rate": 1.4323938637174455e-05,
+    "epoch": 0.7134655474175892,
+    "total_flos": 304361011937280000,
+    "step": 8000
+  },
+  {
+    "loss": 3.45114453125,
+    "learning_rate": 1.209418480199786e-05,
+    "epoch": 0.7580571441311885,
+    "total_flos": 323383575183360000,
+    "step": 8500
+  },
+  {
+    "loss": 3.4411015625,
+    "learning_rate": 9.864430966821263e-06,
+    "epoch": 0.8026487408447878,
+    "total_flos": 342406138429440000,
+    "step": 9000
+  },
+  {
+    "loss": 3.44307421875,
+    "learning_rate": 7.634677131644667e-06,
+    "epoch": 0.8472403375583871,
+    "total_flos": 361428701675520000,
+    "step": 9500
+  },
+  {
+    "loss": 3.44175,
+    "learning_rate": 5.40492329646807e-06,
+    "epoch": 0.8918319342719865,
+    "total_flos": 380451264921600000,
+    "step": 10000
+  },
+  {
+    "loss": 3.43734375,
+    "learning_rate": 3.175169461291474e-06,
+    "epoch": 0.9364235309855857,
+    "total_flos": 399473828167680000,
+    "step": 10500
+  },
+  {
+    "loss": 3.4352890625,
+    "learning_rate": 9.45415626114877e-07,
+    "epoch": 0.9810151276991851,
+    "total_flos": 418496391413760000,
+    "step": 11000
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c28843f24b1a5aae6def4ca20d454bfe889c4ef68547c69810655c110909ba6f
+size 3134064907

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"model_max_length": 1024}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebc1f4880f3c3a058ac3cd8262845d240baf128bde3e3b7cc5bfbd05b9b7be05
+size 1839