Update README.md

README.md

@@ -1,3 +1,40 @@
 ---
 license: other
 ---
+
+```python
+#!/bin/bash
+# Apache v2 license
+# Copyright (C) 2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Teacher Preparation
+
+# Notes:
+# Auto mixed precision can be used by adding --fp16
+# Distributed training can be used with the torch.distributed.lauch app
+
+TEACHER_PATH=./bert-base-uncased-teacher-preparation-pretrain
+OUTPUT_DIR=$TEACHER_PATH
+DATA_CACHE_DIR=/root/kaokao/Model-Compression-Research-Package/examples/transformers/language-modeling/wikipedia_processed_for_pretrain
+
+python -m torch.distributed.launch \
+    --nproc_per_node=8 \
+    ../../examples/transformers/language-modeling/run_mlm.py \
+    --model_name_or_path bert-base-uncased \
+    --datasets_name_config wikipedia:20200501.en \
+    --data_process_type segment_pair_nsp \
+    --dataset_cache_dir $DATA_CACHE_DIR \
+    --do_train \
+    --learning_rate 5e-5 \
+    --max_steps 100000 \
+    --warmup_ratio 0.01 \
+    --weight_decay 0.01 \
+    --per_device_train_batch_size 8 \
+    --gradient_accumulation_steps 4 \
+    --logging_steps 10 \
+    --save_steps 5000 \
+    --save_total_limit 2 \
+    --output_dir $OUTPUT_DIR \
+    --run_name pofa-teacher-prepare-pretrain
+```