FEAT: first commit

.gitattributes

@@ -30,3 +30,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,49 @@
 ---
+language: 
+  - zh
 license: apache-2.0
+tags:
+  - bart
+
+widget:
+- text: "桂林是著名的[MASK]，它有很多[MASK]。"
+
+
 ---
+# Randeng-BART-759M-BertTokenizer model (Chinese)，one model of [Fengshenbang-LM](https://github.com/IDEA-CCNL/Fengshenbang-LM)
+
+The 759M million parameter Randeng-BART large model, using 180G Chinese data, 8 A100(40G) training for 7 days，which is a Encoder-Only transformer structure.
+
+We use bert vocab as our tokenizer.
+
+## Task Description
+
+Randeng-BART-759M-BertTokenizer is pre-trained by Text-Infilling task from BART [paper](https://readpaper.com/pdf-annotate/note?noteId=675945911766249472&pdfId=550970997159968917)
+
+You can find our pretrain's code in [Fengshengbang-LM](https://github.com/IDEA-CCNL/Fengshenbang-LM/tree/main/fengshen/examples/pretrain_randeng_bart)
+
+## Usage
+
+```python
+from transformers import BartForConditionalGeneration, AutoTokenizer, Text2TextGenerationPipeline
+import torch
+
+tokenizer=AutoTokenizer.from_pretrained('IDEA-CCNL/Randeng-BART-759M-BertTokenizer', use_fast=false)
+model=BartForConditionalGeneration.from_pretrained('IDEA-CCNL/Randeng-BART-759M-BertTokenizer')
+text = '桂林是著名的[MASK]，它有很多[MASK]。'
+text2text_generator = Text2TextGenerationPipeline(model, tokenizer)
+print(text2text_generator(text, max_length=50, do_sample=False))
+```
+
+## Citation
+
+If you find the resource is useful, please cite the following website in your paper.
+
+```
+@misc{Fengshenbang-LM,
+  title={Fengshenbang-LM},
+  author={IDEA-CCNL},
+  year={2022},
+  howpublished={\url{https://github.com/IDEA-CCNL/Fengshenbang-LM}},
+}
+```

config.json

@@ -0,0 +1,44 @@
+{
+    "_name_or_path": "bart-759M",
+    "activation_dropout": 0.1,
+    "activation_function": "gelu",
+    "add_bias_logits": false,
+    "add_final_layer_norm": false,
+    "architectures": [
+        "BartForConditionalGeneration"
+    ],
+    "attention_dropout": 0.1,
+    "bos_token_id": 101,
+    "classif_dropout": 0.1,
+    "classifier_dropout": 0.0,
+    "d_model": 1024,
+    "d_kv": 64,
+    "decoder_attention_heads": 16,
+    "decoder_ffn_dim": 4096,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 24,
+    "decoder_start_token_id": 102,
+    "dropout": 0.1,
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 24,
+    "eos_token_id": 102,
+    "forced_eos_token_id": 102,
+    "num_labels": 3,
+    "init_std": 0.02,
+    "is_encoder_decoder": true,
+    "max_position_embeddings": 1024,
+    "model_type": "bart",
+    "no_repeat_ngram_size": 3,
+    "normalize_before": false,
+    "normalize_embedding": true,
+    "num_beams": 4,
+    "num_hidden_layers": 24,
+    "pad_token_id": 0,
+    "scale_embedding": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.16.0.dev0",
+    "use_cache": true,
+    "vocab_size": 50265
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff970b3d11a9fa3c7a4d3f7a8694f0da3e56fb43605f73b1960a0c308d5237c4
+size 1518240291

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+    "do_lower_case": true,
+    "do_basic_tokenize": true,
+    "never_split": null,
+    "unk_token": "[UNK]",
+    "sep_token": "[SEP]",
+    "pad_token": "[PAD]",
+    "cls_token": "[CLS]",
+    "mask_token": "[MASK]",
+    "tokenize_chinese_chars": true,
+    "strip_accents": null,
+    "special_tokens_map_file": null,
+    "name_or_path": "/cognitive_comp/gaoxinyu/pretrained_model/bert-1.3B",
+    "tokenizer_class": "BertTokenizer"
+}