diff --git a/cpm-bee-10b.json b/cpm-bee-10b.json new file mode 100644 index 0000000000000000000000000000000000000000..c34b2e0afb58f8ecd165210b77eb3086d5ad3994 --- /dev/null +++ b/cpm-bee-10b.json @@ -0,0 +1,14 @@ +{ + "vocab_size": 86583, + "dim_model": 4096, + "dim_ff" : 10240, + "num_layers" : 48, + "num_heads": 32, + "dim_head" : 128, + "dropout_p" : 0.0, + "position_bias_num_buckets" : 256, + "position_bias_num_segment_buckets": 256, + "position_bias_max_distance" : 2048, + "eps" : 1e-6, + "half" : true +} diff --git a/vocab.txt b/vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..b977bdaf9898e679a19e338be19321bc1d6f7ede --- /dev/null +++ b/vocab.txt @@ -0,0 +1,86583 @@ + + + + + + + + + + +