Model save

README.md

@@ -1,23 +1,9 @@
 ---
-base_model: fnlp/bert-base-chinese
 tags:
 - generated_from_trainer
 model-index:
 - name: bart-base-cantonese
   results: []
-datasets:
-- indiejoseph/wikipedia-zh-yue-filtered
-- indiejoseph/cc100-yue
-- indiejoseph/ted-transcriptions-cantonese
-- indiejoseph/c4-cantonese-filtered
-- mozilla-foundation/common_voice_13_0
-- jed351/rthk_news
-- jed351/shikoto_zh_hk
-widget:
-- text: "今日去咗旺角[MASK]"
-  example_title: "Mong Kok"
-- text: "今時今日香港係一個[MASK]。"
-  example_title: "Hong Kong"
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -25,9 +11,11 @@ should probably proofread and complete it, then remove this comment. -->
 
 # bart-base-cantonese
 
-This model is a continue pre-train version of [fnlp/bart-base-chinese](https://huggingface.co/fnlp/bart-base-chinese) on filtered Cantonese common crawl dataset with 950M tokens.
+This model was trained from scratch on the None dataset.
 
-This tokenizer has extended the Bert tokenizer from fnlp/bart-base-chinese with 500 more Chinese characters commonly found in Cantonese
+## Model description
+
+More information needed
 
 ## Intended uses & limitations
 
@@ -57,7 +45,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.35.0.dev0
+- Transformers 4.35.2
 - Pytorch 2.1.1+cu121
 - Datasets 2.14.6
 - Tokenizers 0.14.1

all_results.json

@@ -1,15 +1,15 @@
 {
-    "epoch": 3.0,
-    "eval_accuracy": 0.8416168286663981,
-    "eval_loss": 0.8037419319152832,
-    "eval_runtime": 188.7732,
-    "eval_samples": 6205,
-    "eval_samples_per_second": 32.87,
-    "eval_steps_per_second": 2.055,
-    "perplexity": 2.233884351646835,
-    "train_loss": 0.9618661650365136,
-    "train_runtime": 9694.4976,
-    "train_samples": 63336,
-    "train_samples_per_second": 19.6,
-    "train_steps_per_second": 1.225
+    "epoch": 1.0,
+    "eval_accuracy": 0.8386325284846604,
+    "eval_loss": 0.8430067300796509,
+    "eval_runtime": 339.0879,
+    "eval_samples": 11278,
+    "eval_samples_per_second": 33.26,
+    "eval_steps_per_second": 2.079,
+    "perplexity": 2.3233421480194,
+    "train_loss": 0.787189019458772,
+    "train_runtime": 32945.1529,
+    "train_samples": 648171,
+    "train_samples_per_second": 19.674,
+    "train_steps_per_second": 1.23
 }

config.json

@@ -69,7 +69,7 @@
   },
   "tokenizer_class": "BertTokenizer",
   "torch_dtype": "float32",
-  "transformers_version": "4.35.0.dev0",
+  "transformers_version": "4.35.2",
   "use_cache": true,
   "vocab_size": 51371
 }

eval_results.json

@@ -1,10 +1,5 @@
 {
-    "epoch": 3.0,
-    "eval_accuracy": 0.8416168286663981,
-    "eval_loss": 0.8037419319152832,
-    "eval_runtime": 188.7732,
-    "eval_samples": 6205,
-    "eval_samples_per_second": 32.87,
-    "eval_steps_per_second": 2.055,
-    "perplexity": 2.233884351646835
+    "eval_accuracy": 0.0,
+    "eval_loss": 11.697545369466146,
+    "eval_perplexity": 120276.11891132068
 }

generation_config.json

@@ -8,5 +8,5 @@
   "no_repeat_ngram_size": 3,
   "num_beams": 4,
   "pad_token_id": 0,
-  "transformers_version": "4.35.0.dev0"
+  "transformers_version": "4.35.2"
 }

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cd8dc13b00393f4be5ae61eea43edb23b5f9cc6437f8979391a0a677f2519d1
+size 561314676

runs/Nov28_14-00-45_44d3998eda2b/events.out.tfevents.1701180578.44d3998eda2b.3099222.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1de4ee578d88c3ccacef55afb9a31c2f9e2f78be1ae90047244709a6ff0520f9
+size 5428

runs/Nov28_14-24-09_44d3998eda2b/events.out.tfevents.1701183372.44d3998eda2b.3101657.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:633b472eb60ffec309488358ef2154daac75bf51ebb63e5c5b0c9344d401dcb2
+size 9981

runs/Nov28_18-18-01_44d3998eda2b/events.out.tfevents.1701197416.44d3998eda2b.3136596.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d870af4911b82236fc110d03aded1716e3e54bc74e8d10b3178bce21c0b6cfee
+size 5899

runs/Nov28_19-12-15_44d3998eda2b/events.out.tfevents.1701200666.44d3998eda2b.3141349.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dc98660544a599d020a0670457cdf57aa621524f6608c822116daeb79e6843f
+size 18811

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.0,
-    "train_loss": 0.9618661650365136,
-    "train_runtime": 9694.4976,
-    "train_samples": 63336,
-    "train_samples_per_second": 19.6,
-    "train_steps_per_second": 1.225
+    "epoch": 1.0,
+    "train_loss": 0.787189019458772,
+    "train_runtime": 32945.1529,
+    "train_samples": 648171,
+    "train_samples_per_second": 19.674,
+    "train_steps_per_second": 1.23
 }

trainer_state.json

@@ -1,166 +1,514 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.0,
+  "epoch": 1.0,
   "eval_steps": 2000.0,
-  "global_step": 11877,
+  "global_step": 40511,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.13,
-      "learning_rate": 2.5e-05,
-      "loss": 1.5023,
+      "epoch": 0.01,
+      "learning_rate": 6.16979269496545e-06,
+      "loss": 1.3917,
       "step": 500
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 5e-05,
-      "loss": 1.124,
+      "epoch": 0.02,
+      "learning_rate": 1.23395853899309e-05,
+      "loss": 1.0351,
       "step": 1000
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 7.500000000000001e-05,
-      "loss": 1.0616,
+      "epoch": 0.04,
+      "learning_rate": 1.8509378084896346e-05,
+      "loss": 0.9747,
       "step": 1500
     },
     {
-      "epoch": 0.51,
-      "learning_rate": 0.0001,
-      "loss": 1.0334,
+      "epoch": 0.05,
+      "learning_rate": 2.46791707798618e-05,
+      "loss": 0.9373,
       "step": 2000
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 9.49377341297965e-05,
-      "loss": 1.0149,
+      "epoch": 0.06,
+      "learning_rate": 3.084896347482725e-05,
+      "loss": 0.9124,
       "step": 2500
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 8.9875468259593e-05,
-      "loss": 0.9938,
+      "epoch": 0.07,
+      "learning_rate": 3.701875616979269e-05,
+      "loss": 0.8946,
       "step": 3000
     },
     {
-      "epoch": 0.88,
-      "learning_rate": 8.481320238938948e-05,
-      "loss": 0.987,
+      "epoch": 0.09,
+      "learning_rate": 4.318854886475814e-05,
+      "loss": 0.8819,
       "step": 3500
     },
     {
-      "epoch": 1.01,
-      "learning_rate": 7.975093651918599e-05,
-      "loss": 0.97,
+      "epoch": 0.1,
+      "learning_rate": 4.93583415597236e-05,
+      "loss": 0.8723,
       "step": 4000
     },
     {
-      "epoch": 1.14,
-      "learning_rate": 7.468867064898249e-05,
-      "loss": 0.9497,
+      "epoch": 0.11,
+      "learning_rate": 4.9385611234537426e-05,
+      "loss": 0.8615,
       "step": 4500
     },
     {
-      "epoch": 1.26,
-      "learning_rate": 6.962640477877899e-05,
-      "loss": 0.9438,
+      "epoch": 0.12,
+      "learning_rate": 4.869990948736938e-05,
+      "loss": 0.8553,
       "step": 5000
     },
     {
-      "epoch": 1.39,
-      "learning_rate": 6.456413890857548e-05,
-      "loss": 0.938,
+      "epoch": 0.14,
+      "learning_rate": 4.801420774020132e-05,
+      "loss": 0.8466,
       "step": 5500
     },
     {
-      "epoch": 1.52,
-      "learning_rate": 5.950187303837198e-05,
-      "loss": 0.9261,
+      "epoch": 0.15,
+      "learning_rate": 4.732850599303327e-05,
+      "loss": 0.8412,
       "step": 6000
     },
     {
-      "epoch": 1.64,
-      "learning_rate": 5.443960716816847e-05,
-      "loss": 0.924,
+      "epoch": 0.16,
+      "learning_rate": 4.6642804245865215e-05,
+      "loss": 0.8331,
       "step": 6500
     },
     {
-      "epoch": 1.77,
-      "learning_rate": 4.937734129796497e-05,
-      "loss": 0.9165,
+      "epoch": 0.17,
+      "learning_rate": 4.5957102498697166e-05,
+      "loss": 0.8252,
       "step": 7000
     },
     {
-      "epoch": 1.89,
-      "learning_rate": 4.431507542776147e-05,
-      "loss": 0.9034,
+      "epoch": 0.19,
+      "learning_rate": 4.527140075152912e-05,
+      "loss": 0.8164,
       "step": 7500
     },
     {
-      "epoch": 2.02,
-      "learning_rate": 3.925280955755796e-05,
-      "loss": 0.8966,
+      "epoch": 0.2,
+      "learning_rate": 4.458569900436107e-05,
+      "loss": 0.8139,
       "step": 8000
     },
     {
-      "epoch": 2.15,
-      "learning_rate": 3.419054368735446e-05,
-      "loss": 0.8829,
+      "epoch": 0.21,
+      "learning_rate": 4.389999725719302e-05,
+      "loss": 0.8072,
       "step": 8500
     },
     {
-      "epoch": 2.27,
-      "learning_rate": 2.9128277817150957e-05,
-      "loss": 0.8843,
+      "epoch": 0.22,
+      "learning_rate": 4.321429551002496e-05,
+      "loss": 0.8096,
       "step": 9000
     },
     {
-      "epoch": 2.4,
-      "learning_rate": 2.4066011946947456e-05,
-      "loss": 0.8725,
+      "epoch": 0.23,
+      "learning_rate": 4.252859376285691e-05,
+      "loss": 0.8026,
       "step": 9500
     },
     {
-      "epoch": 2.53,
-      "learning_rate": 1.900374607674395e-05,
-      "loss": 0.8712,
+      "epoch": 0.25,
+      "learning_rate": 4.1842892015688856e-05,
+      "loss": 0.8027,
       "step": 10000
     },
     {
-      "epoch": 2.65,
-      "learning_rate": 1.3941480206540447e-05,
-      "loss": 0.8673,
+      "epoch": 0.26,
+      "learning_rate": 4.115719026852081e-05,
+      "loss": 0.7952,
       "step": 10500
     },
     {
-      "epoch": 2.78,
-      "learning_rate": 8.879214336336945e-06,
-      "loss": 0.8669,
+      "epoch": 0.27,
+      "learning_rate": 4.047148852135275e-05,
+      "loss": 0.7936,
       "step": 11000
     },
     {
-      "epoch": 2.9,
-      "learning_rate": 3.816948466133442e-06,
-      "loss": 0.8679,
+      "epoch": 0.28,
+      "learning_rate": 3.97857867741847e-05,
+      "loss": 0.7913,
       "step": 11500
     },
     {
-      "epoch": 3.0,
-      "step": 11877,
-      "total_flos": 5.792740247863296e+16,
-      "train_loss": 0.9618661650365136,
-      "train_runtime": 9694.4976,
-      "train_samples_per_second": 19.6,
-      "train_steps_per_second": 1.225
+      "epoch": 0.3,
+      "learning_rate": 3.910008502701665e-05,
+      "loss": 0.7896,
+      "step": 12000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.8414383279848596e-05,
+      "loss": 0.7857,
+      "step": 12500
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.7728681532680547e-05,
+      "loss": 0.7842,
+      "step": 13000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.70429797855125e-05,
+      "loss": 0.7813,
+      "step": 13500
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.635727803834445e-05,
+      "loss": 0.7812,
+      "step": 14000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.567157629117639e-05,
+      "loss": 0.7793,
+      "step": 14500
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.498587454400834e-05,
+      "loss": 0.7721,
+      "step": 15000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.4300172796840286e-05,
+      "loss": 0.7726,
+      "step": 15500
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.361447104967224e-05,
+      "loss": 0.7714,
+      "step": 16000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.292876930250419e-05,
+      "loss": 0.7737,
+      "step": 16500
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.224306755533613e-05,
+      "loss": 0.7661,
+      "step": 17000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.155736580816808e-05,
+      "loss": 0.7675,
+      "step": 17500
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.0871664061000026e-05,
+      "loss": 0.7688,
+      "step": 18000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.0185962313831976e-05,
+      "loss": 0.7637,
+      "step": 18500
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9500260566663924e-05,
+      "loss": 0.7644,
+      "step": 19000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.8814558819495874e-05,
+      "loss": 0.7632,
+      "step": 19500
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.8128857072327825e-05,
+      "loss": 0.7607,
+      "step": 20000
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.744315532515977e-05,
+      "loss": 0.7614,
+      "step": 20500
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.675745357799172e-05,
+      "loss": 0.7602,
+      "step": 21000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6071751830823667e-05,
+      "loss": 0.7564,
+      "step": 21500
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5386050083655617e-05,
+      "loss": 0.7574,
+      "step": 22000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4700348336487565e-05,
+      "loss": 0.7534,
+      "step": 22500
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.4014646589319512e-05,
+      "loss": 0.7567,
+      "step": 23000
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.332894484215146e-05,
+      "loss": 0.7534,
+      "step": 23500
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.2643243094983406e-05,
+      "loss": 0.7555,
+      "step": 24000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.1957541347815357e-05,
+      "loss": 0.7534,
+      "step": 24500
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.1271839600647304e-05,
+      "loss": 0.7475,
+      "step": 25000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.058613785347925e-05,
+      "loss": 0.7482,
+      "step": 25500
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.99004361063112e-05,
+      "loss": 0.7475,
+      "step": 26000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.9214734359143146e-05,
+      "loss": 0.7479,
+      "step": 26500
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.8529032611975096e-05,
+      "loss": 0.7433,
+      "step": 27000
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.7843330864807044e-05,
+      "loss": 0.7473,
+      "step": 27500
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.7157629117638994e-05,
+      "loss": 0.7439,
+      "step": 28000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.647192737047094e-05,
+      "loss": 0.7423,
+      "step": 28500
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.578622562330289e-05,
+      "loss": 0.7429,
+      "step": 29000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.5100523876134836e-05,
+      "loss": 0.7424,
+      "step": 29500
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4414822128966785e-05,
+      "loss": 0.7408,
+      "step": 30000
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.3729120381798732e-05,
+      "loss": 0.7402,
+      "step": 30500
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.3043418634630683e-05,
+      "loss": 0.7376,
+      "step": 31000
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.235771688746263e-05,
+      "loss": 0.7371,
+      "step": 31500
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1672015140294577e-05,
+      "loss": 0.741,
+      "step": 32000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.0986313393126526e-05,
+      "loss": 0.7354,
+      "step": 32500
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0300611645958475e-05,
+      "loss": 0.7343,
+      "step": 33000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.614909898790423e-06,
+      "loss": 0.7357,
+      "step": 33500
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.92920815162237e-06,
+      "loss": 0.7319,
+      "step": 34000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.243506404454319e-06,
+      "loss": 0.7339,
+      "step": 34500
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.557804657286268e-06,
+      "loss": 0.732,
+      "step": 35000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.872102910118215e-06,
+      "loss": 0.7352,
+      "step": 35500
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.186401162950164e-06,
+      "loss": 0.7364,
+      "step": 36000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.500699415782112e-06,
+      "loss": 0.733,
+      "step": 36500
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.814997668614059e-06,
+      "loss": 0.7318,
+      "step": 37000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 4.129295921446008e-06,
+      "loss": 0.7333,
+      "step": 37500
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.4435941742779558e-06,
+      "loss": 0.7358,
+      "step": 38000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7578924271099047e-06,
+      "loss": 0.7332,
+      "step": 38500
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0721906799418528e-06,
+      "loss": 0.7325,
+      "step": 39000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.3864889327738007e-06,
+      "loss": 0.7282,
+      "step": 39500
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.00787185605749e-07,
+      "loss": 0.731,
+      "step": 40000
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.508543843769714e-08,
+      "loss": 0.7318,
+      "step": 40500
+    },
+    {
+      "epoch": 1.0,
+      "step": 40511,
+      "total_flos": 1.9760674493693952e+17,
+      "train_loss": 0.787189019458772,
+      "train_runtime": 32945.1529,
+      "train_samples_per_second": 19.674,
+      "train_steps_per_second": 1.23
     }
   ],
   "logging_steps": 500,
-  "max_steps": 11877,
-  "num_train_epochs": 3,
+  "max_steps": 40511,
+  "num_train_epochs": 1,
   "save_steps": 2000,
-  "total_flos": 5.792740247863296e+16,
+  "total_flos": 1.9760674493693952e+17,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b73eb5f3d3b5c7077b84e7429b72db499f4db630cec7723c33cab47fcfc7056
-size 4536
+oid sha256:f321738d7dc9125e15c79e15f46839e735df9d313bee21b88fbd1a2628cf64c1
+size 4664