Update README.md

README.md

@@ -38,42 +38,43 @@ The model is pre-trained by [UER-py](https://github.com/dbiir/UER-py/) on [Tence
 ```
 python3 preprocess.py --corpus_path corpora/cluecorpussmall.txt \
                       --vocab_path models/google_zh_vocab.txt \
-                      --dataset_path cluecorpussmall_bart_seq512_dataset.pt \
+                      --dataset_path cluecorpussmall_pegasus_seq512_dataset.pt \
                       --processes_num 32 --seq_length 512 \
-                      --dynamic_masking --target bart 
+                      --target gsg --sentence_selection_strategy random
 ```
 
 ```
-python3 pretrain.py --dataset_path cluecorpussmall_bart_seq512_dataset.pt \
+python3 pretrain.py --dataset_path cluecorpussmall_pegasus_seq512_dataset.pt \
                     --vocab_path models/google_zh_vocab.txt \
-                    --config_path models/bart/base_config.json \
-                    --output_model_path models/cluecorpussmall_bart_base_seq512_model.bin \
+                    --config_path models/pegasus/base_config.json \
+                    --output_model_path models/cluecorpussmall_pegasus_base_seq512_model.bin \
                     --world_size 8 --gpu_ranks 0 1 2 3 4 5 6 7 \
                     --total_steps 1000000 --save_checkpoint_steps 100000 --report_steps 50000 \
-                    --learning_rate 1e-4 --batch_size 16 \
-                    --span_masking --span_max_length 3 \
-                    --embedding word_pos --tgt_embedding word_pos \
-                    --encoder transformer --mask fully_visible --decoder transformer \
-                    --target bart --tie_weights --has_lmtarget_bias
+                    --learning_rate 1e-4 --batch_size 8 \
+                    --embedding word_sinusoidalpos --remove_embedding_layernorm --tgt_embedding word_sinusoidalpos \
+                    --encoder transformer --mask fully_visible --layernorm_positioning pre --decoder transformer \
+                    --target gsg --tie_weights --has_lmtarget_bias
 ```
 
 Finally, we convert the pre-trained model into Huggingface's format:
 
 ```
-python3 scripts/convert_bart_from_uer_to_huggingface.py --input_model_path cluecorpussmall_bart_base_seq512_model.bin-250000 \                                                                
-                                                        --output_model_path pytorch_model.bin \                                                                                            
-                                                        --layers_num 6
+python3 scripts/convert_pegasus_from_uer_to_huggingface.py --input_model_path cluecorpussmall_pegasus_base_seq512_model.bin-250000 \                                                                
+                                                           --output_model_path pytorch_model.bin \                                                                                            
+                                                           --layers_num 12
 ```
 
 
 ### BibTeX entry and citation info
 
 ```
-@article{lewis2019bart,
-  title={Bart: Denoising sequence-to-sequence pre-training for natural language generation, translation, and comprehension},
-  author={Lewis, Mike and Liu, Yinhan and Goyal, Naman and Ghazvininejad, Marjan and Mohamed, Abdelrahman and Levy, Omer and Stoyanov, Ves and Zettlemoyer, Luke},
-  journal={arXiv preprint arXiv:1910.13461},
-  year={2019}
+@inproceedings{zhang2020pegasus,
+  title={Pegasus: Pre-training with extracted gap-sentences for abstractive summarization},
+  author={Zhang, Jingqing and Zhao, Yao and Saleh, Mohammad and Liu, Peter},
+  booktitle={International Conference on Machine Learning},
+  pages={11328--11339},
+  year={2020},
+  organization={PMLR}
 }
 
 @article{zhao2019uer,