Hugging Face's logo

microsoft
/
unihanlm-base

Feature Extraction Transformers PyTorch TensorFlow
Chinese Japanese xlm crosslingual Inference Endpoints
Model card Files Community
1
unihanlm-base
File size: 4,243 Bytes 
1eb3dc6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
{
  "accumulate_gradients": 1,
  "ae_steps": [],
  "amp": 1,
  "architectures": [
    "XLMModel"
  ],
  "asm": false,
  "attention_dropout": 0.1,
  "batch_size": 64,
  "beam_size": 1,
  "bos_index": 0,
  "bos_token_id": 0,
  "bptt": 256,
  "bt_src_langs": [],
  "bt_steps": [],
  "causal": false,
  "clip_grad_norm": 5,
  "clm_steps": [],
  "command": "python train.py --local_rank=0 --exp_name unihan_zh_ja --dump_path '/mnt/exp/ft_char' --data_path 'data/processed/xlm_zh_ja/new' --lgs 'zh-ja' --clm_steps '' --mlm_steps 'ja,zh' --emb_dim 1024 --n_layers 12 --n_heads 16 --dropout '0.1' --attention_dropout '0.1' --gelu_activation true --batch_size 64 --bptt 256 --optimizer 'adam_inverse_sqrt,lr=0.00005,warmup_updates=30000,beta1=0.9,beta2=0.999,weight_decay=0.01,eps=0.000001' --epoch_size 300000 --max_epoch 100000 --validation_metrics _valid_mlm_ppl --stopping_criterion '_valid_mlm_ppl,25' --fp16 true --amp 1 --exp_id epoch169 --reload_model '/mnt/exp/hard_pretrain/unihan_zh_ja/recycled/converted-best-valid_mlm_ppl.pth' --exp_id \"epoch169\"",
  "context_size": 0,
  "data_path": "data/processed/xlm_zh_ja/new",
  "debug": false,
  "debug_slurm": false,
  "debug_train": false,
  "dropout": 0.1,
  "dump_path": "/mnt/exp/ft_char/unihan_zh_ja/epoch169",
  "emb_dim": 1024,
  "embed_init_std": 0.02209708691207961,
  "encoder_only": true,
  "end_n_top": 5,
  "eos_index": 1,
  "epoch_size": 300000,
  "eval_bleu": false,
  "eval_only": false,
  "exp_id": "epoch169",
  "exp_name": "unihan_zh_ja",
  "fp16": true,
  "gelu_activation": true,
  "global_rank": 0,
  "group_by_size": true,
  "hyp_path": "/mnt/exp/ft_char/unihan_zh_ja/epoch169/hypotheses",
  "id2lang": {
    "0": "ja",
    "1": "zh"
  },
  "init_std": 0.02,
  "is_encoder": true,
  "is_master": true,
  "is_slurm_job": false,
  "lambda_ae": 1.0,
  "lambda_ae_config": null,
  "lambda_bt": 1.0,
  "lambda_bt_config": null,
  "lambda_clm": 1.0,
  "lambda_clm_config": null,
  "lambda_mlm": 1.0,
  "lambda_mlm_config": null,
  "lambda_mt": 1.0,
  "lambda_mt_config": null,
  "lambda_pc": 1.0,
  "lambda_pc_config": null,
  "lang2id": {
    "ja": 0,
    "zh": 1
  },
  "lang_id": 0,
  "langs": [
    "zh",
    "ja"
  ],
  "layer_norm_eps": 1e-12,
  "lg_sampling_factor": -1,
  "lgs": "zh-ja",
  "local_rank": 0,
  "mask_index": 5,
  "mask_token_id": 0,
  "master_port": -1,
  "max_batch_size": 0,
  "max_epoch": 100000,
  "max_len": 100,
  "max_position_embeddings": 512,
  "max_vocab": -1,
  "min_count": 0,
  "mlm_steps": [
    [
      "ja",
      null
    ],
    [
      "zh",
      null
    ]
  ],
  "model_type": "xlm",
  "mono_dataset": {
    "ja": {
      "test": "data/processed/xlm_zh_ja/new/test.ja.pth",
      "train": "data/processed/xlm_zh_ja/new/train.ja.pth",
      "valid": "data/processed/xlm_zh_ja/new/valid.ja.pth"
    },
    "zh": {
      "test": "data/processed/xlm_zh_ja/new/test.zh.pth",
      "train": "data/processed/xlm_zh_ja/new/train.zh.pth",
      "valid": "data/processed/xlm_zh_ja/new/valid.zh.pth"
    }
  },
  "mt_steps": [],
  "multi_gpu": true,
  "multi_node": false,
  "n_gpu_per_node": 8,
  "n_heads": 16,
  "n_langs": 2,
  "n_layers": 12,
  "n_nodes": 1,
  "node_id": 0,
  "optimizer": "adam_inverse_sqrt,lr=0.00005,warmup_updates=30000,beta1=0.9,beta2=0.999,weight_decay=0.01,eps=0.000001",
  "pad_index": 2,
  "pad_token_id": 2,
  "para_dataset": {},
  "pc_steps": [],
  "ref_paths": {},
  "reload_checkpoint": "",
  "reload_emb": "",
  "reload_model": "/mnt/exp/hard_pretrain/unihan_zh_ja/recycled/converted-best-valid_mlm_ppl.pth",
  "sample_alpha": 0,
  "save_periodic": 0,
  "share_inout_emb": true,
  "sinusoidal_embeddings": false,
  "split_data": false,
  "start_n_top": 5,
  "stopping_criterion": "_valid_mlm_ppl,25",
  "summary_activation": null,
  "summary_first_dropout": 0.1,
  "summary_proj_to_labels": true,
  "summary_type": "first",
  "summary_use_proj": true,
  "tokens_per_batch": -1,
  "unk_index": 3,
  "use_lang_emb": true,
  "use_memory": false,
  "validation_metrics": "_valid_mlm_ppl",
  "vocab_size": 24044,
  "word_blank": 0,
  "word_dropout": 0,
  "word_keep": 0.1,
  "word_mask": 0.8,
  "word_mask_keep_rand": "0.8,0.1,0.1",
  "word_pred": 0.15,
  "word_rand": 0.1,
  "word_shuffle": 0,
  "world_size": 8
}