Repository: FlagAI-Open/FlagAI (language: python)

Issue:
fix bugs in evaluate



Discussion / hints:

Fix patch:
diff --git a/doc_zh/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md b/doc_zh/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md
index e5f948c9..d00cca23 100644
--- a/doc_zh/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md
+++ b/doc_zh/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md
@@ -118,7 +118,7 @@ class GLMTitleGenerationCollateFN():
 ```python
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
-my_collate_fn = GLMTitleGenerationCollateFN(pad_id=tokenizer.get_command('pad').Id)
+my_collate_fn = GLMTitleGenerationCollateFN(pad_id=tokenizer.get_command_id('pad'))
 train_dataset = GLMTitleGenerationDataset(train_src,
                                    train_tgt)
 ```
diff --git a/doc_zh/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md b/doc_zh/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md
index 50208993..12ced114 100644
--- a/doc_zh/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md
+++ b/doc_zh/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md
@@ -131,7 +131,7 @@ class GLMPoetryDynamicCollateFN():
 ```python
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
-my_collate_fn = GLMPoetryDynamicCollateFN(pad_id=tokenizer.get_command('pad').Id)
+my_collate_fn = GLMPoetryDynamicCollateFN(pad_id=tokenizer.get_command_id('pad'))
 train_dataset = GLMPoetryDataset(train_src,
                                    train_tgt)
 ```
diff --git a/docs/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md b/docs/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md
index 02488891..71cdba97 100644
--- a/docs/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md
+++ b/docs/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md
@@ -119,7 +119,7 @@ class GLMTitleGenerationCollateFN():
 ```python
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
-my_collate_fn = GLMTitleGenerationCollateFN(pad_id=tokenizer.get_command('pad').Id)
+my_collate_fn = GLMTitleGenerationCollateFN(pad_id=tokenizer.get_command_id('pad'))
 train_dataset = GLMTitleGenerationDataset(train_src,
                                    train_tgt)
 ```
diff --git a/docs/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md b/docs/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md
index aac9d2b0..1f5ec00d 100644
--- a/docs/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md
+++ b/docs/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md
@@ -122,7 +122,7 @@ class GLMPoetryDynamicCollateFN():
 ```python
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
-my_collate_fn = GLMPoetryDynamicCollateFN(pad_id=tokenizer.get_command('pad').Id)
+my_collate_fn = GLMPoetryDynamicCollateFN(pad_id=tokenizer.get_command_id('pad'))
 train_dataset = GLMPoetryDataset(train_src,
                                    train_tgt)
 ```
diff --git a/examples/bert_title_generation_english/generate.py b/examples/bert_title_generation_english/generate.py
index 4cab6e8b..1124d16d 100755
--- a/examples/bert_title_generation_english/generate.py
+++ b/examples/bert_title_generation_english/generate.py
@@ -14,7 +14,7 @@
 maxlen = 512
 auto_loader = AutoLoader(
     "seq2seq",
-    model_name="bert-base-uncased",
+    model_name="BERT-base-en",
     model_dir=model_dir,
 )
 model = auto_loader.get_model()
diff --git a/examples/clip/CLIP.png b/examples/clip/CLIP.png
new file mode 100644
index 00000000..a1b5ec91
Binary files /dev/null and b/examples/clip/CLIP.png differ
diff --git a/examples/clip/data/img/0.jpg b/examples/clip/data/img/0.jpg
new file mode 100644
index 00000000..ce956f45
Binary files /dev/null and b/examples/clip/data/img/0.jpg differ
diff --git a/examples/clip/data/img/1.jpg b/examples/clip/data/img/1.jpg
new file mode 100644
index 00000000..66b8c1d8
Binary files /dev/null and b/examples/clip/data/img/1.jpg differ
diff --git a/examples/clip/data/pairs.csv b/examples/clip/data/pairs.csv
new file mode 100644
index 00000000..2e1a9b2f
--- /dev/null
+++ b/examples/clip/data/pairs.csv
@@ -0,0 +1,3 @@
+title	filepath
+a very typical bus station	0.jpg
+the jetty : different types of plants to establish a variety of ecosystems .	1.jpg
diff --git a/examples/clip/deepspeed.json b/examples/clip/deepspeed.json
new file mode 100644
index 00000000..f2339ca3
--- /dev/null
+++ b/examples/clip/deepspeed.json
@@ -0,0 +1,48 @@
+{
+    "train_micro_batch_size_per_gpu": 64,
+    "gradient_accumulation_steps": 1,
+    "steps_per_print": 100,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+      "stage": 2,
+      "contiguous_gradients": false,
+      "overlap_comm": true,
+      "reduce_scatter": true,
+      "reduce_bucket_size": 5e7,
+      "allgather_bucket_size": 5e7,
+      "cpu_offload": true
+    },
+    "scheduler": {
+      "type": "WarmupLR",
+      "params": {
+          "warmup_min_lr": 0,
+          "warmup_max_lr": 1e-5,
+          "warmup_num_steps": 2000
+      }
+   },
+    "zero_allow_untested_optimizer": true,
+    "fp16": {
+      "enabled": true,
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "optimizer": {
+      "type": "Adam",
+      "params": {
+        "lr": 1e-5,
+        "weight_decay": 0.1,
+        "betas": [
+          0.9,
+          0.98
+        ],
+        "eps": 1e-6
+      }
+    },
+    "activation_checkpointing": {
+      "partition_activations": true,
+      "contiguous_memory_optimization": false
+    },
+    "wall_clock_breakdown": false
+  }
diff --git a/examples/clip/hostfile b/examples/clip/hostfile
new file mode 100644
index 00000000..51356577
--- /dev/null
+++ b/examples/clip/hostfile
@@ -0,0 +1,1 @@
+127.0.0.1 slots=2
\ No newline at end of file
diff --git a/examples/clip/inference_clip.py b/examples/clip/inference_clip.py
new file mode 100644
index 00000000..28bf2636
--- /dev/null
+++ b/examples/clip/inference_clip.py
@@ -0,0 +1,30 @@
+import torch
+from PIL import Image
+from flagai.auto_model.auto_loader import AutoLoader
+from flagai.data.dataset.mm.clip_dataset import clip_transform
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+loader = AutoLoader(task_name="txt_img_matching", #contrastive learning
+                    model_name="clip-base-p32-224")
+
+model = loader.get_model()
+model.eval()
+model.to(device)
+tokenizer = loader.get_tokenizer()
+transform = clip_transform(img_size=model.image_size)
+
+def inference():
+    image = Image.open("./CLIP.png")
+    image = transform(image).unsqueeze(0).to(device)
+    text = tokenizer.tokenize_as_tensor(["a diagram", "a dog", "a cat"]).to(device)
+
+    with torch.no_grad():
+        image_features = model.encode_image(image)
+        text_features = model.encode_text(text)
+        text_probs = (image_features @ text_features.T).softmax(dim=-1)
+
+    print(text_probs.cpu().numpy()[0].tolist())
+
+if __name__=="__main__":
+    inference()
diff --git a/examples/clip/train_clip.py b/examples/clip/train_clip.py
new file mode 100644
index 00000000..4e60e212
--- /dev/null
+++ b/examples/clip/train_clip.py
@@ -0,0 +1,36 @@
+import torch
+from flagai.data.dataset.mm.clip_dataset import CsvDataset, clip_transform, collate_fn
+from flagai.trainer import Trainer
+from flagai.auto_model.auto_loader import AutoLoader
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+# cd examples/clip
+data_path = "./data/pairs.csv"
+img_dir = "./data/img"
+
+trainer = Trainer(env_type="pytorch",
+                  epochs=5,
+                  pytorch_device=device,
+                  batch_size=64,
+                  lr=1e-4,
+                  log_interval=10,
+                  )
+
+loader = AutoLoader(task_name="txt_img_matching",#contrastive learning
+                    model_name="clip-base-p32-224",
+                    )
+model = loader.get_model()
+tokenizer = loader.get_tokenizer()
+
+transform = clip_transform(img_size=model.image_size)
+train_dataset = CsvDataset(data_path,
+                            img_dir,
+                            transform,
+                            tokenizer)
+
+optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
+trainer.train(model,
+              optimizer=optimizer,
+              train_dataset=train_dataset,
+              collate_fn=collate_fn)
+
diff --git a/examples/clip/train_clip_deepspeed.py b/examples/clip/train_clip_deepspeed.py
new file mode 100644
index 00000000..2fe7c895
--- /dev/null
+++ b/examples/clip/train_clip_deepspeed.py
@@ -0,0 +1,48 @@
+import torch
+from flagai.data.dataset.mm.clip_dataset import CsvDataset, clip_transform, collate_fn
+from flagai.trainer import Trainer
+from flagai.auto_model.auto_loader import AutoLoader
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+# cd examples/clip
+data_path = "./data/pairs.csv"#"/mnt/datasets/multimodal/ConceptualCaptions/Train_GCC-training_output.csv"
+img_dir = "./data/img"#"/mnt/datasets/multimodal/ConceptualCaptions"
+
+trainer = Trainer(
+    env_type="deepspeed",
+    experiment_name="clip",
+    batch_size=64,
+    num_gpus=2,
+    fp16=True,
+    gradient_accumulation_steps=1,
+    lr=1e-4,
+    weight_decay=1e-5,
+    epochs=5,
+    log_interval=1,
+    load_dir=None,
+    pytorch_device=device,
+    save_dir="clip_deepspeed",
+    save_interval=1000,
+    num_checkpoints=1,
+    hostfile="./deepspeed/hostfile",
+    training_script=__file__,
+    deepspeed_config="./deepspeed.json"
+)
+loader = AutoLoader(task_name="txt_img_matching",#contrastive learning
+                    model_name="clip-base-p32-224",
+                    )
+model = loader.get_model()
+tokenizer = loader.get_tokenizer()
+
+transform = clip_transform(img_size=model.image_size)
+train_dataset = CsvDataset(data_path,
+                            img_dir,
+                            transform,
+                            tokenizer)
+
+optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
+trainer.train(model,
+              optimizer=optimizer,
+              train_dataset=train_dataset,
+              collate_fn=collate_fn)
+
diff --git a/examples/glm_blank_filling/glm_generate_samples.py b/examples/glm_blank_filling/glm_generate_samples.py
index 065e1310..cc8818d6 100755
--- a/examples/glm_blank_filling/glm_generate_samples.py
+++ b/examples/glm_blank_filling/glm_generate_samples.py
@@ -5,16 +5,17 @@
 import torch
 
 from flagai.model.glm_model import GLMModel
-from flagai.data.tokenizer import GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.model.predictor.predictor import Predictor
 if __name__ == "__main__":
     """Main training program."""
     print('Generate Samples')
     # Random seeds for reproducability.
     # Model,
-    model = GLMModel.from_pretrain(model_name='GLM-large-ch',
+    model_name = 'GLM-large-ch'
+    model = GLMModel.from_pretrain(model_name=model_name,
                                    download_path="./state_dict/")
-    tokenizer = GLMLargeChTokenizer()
+    tokenizer = Tokenizer.from_pretrained(model_name)
 
     model.cuda(torch.cuda.current_device())
 
diff --git a/examples/glm_poetry_generation/train.py b/examples/glm_poetry_generation/train.py
index 0a994833..a4699143 100644
--- a/examples/glm_poetry_generation/train.py
+++ b/examples/glm_poetry_generation/train.py
@@ -130,7 +130,7 @@ def __call__(self, batch):
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
 my_collate_fn = GLMPoetryDynamicCollateFN(
-    pad_id=tokenizer.get_command('pad').Id)
+    pad_id=tokenizer.get_command_id('pad'))
 train_dataset = BertSeq2seqDataset(train_src, train_tgt)
 
 trainer.train(model, train_dataset=train_dataset, collate_fn=my_collate_fn)
diff --git a/examples/glm_pretrain/train.py b/examples/glm_pretrain/train.py
index 8d21acdd..4e8cc966 100644
--- a/examples/glm_pretrain/train.py
+++ b/examples/glm_pretrain/train.py
@@ -2,7 +2,7 @@
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
 
-from flagai.data.tokenizer import GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.model.glm_model import GLMForSeq2Seq
 from flagai.trainer import Trainer
 from flagai.data.dataset import ConstructBlockStrategy
@@ -24,14 +24,11 @@
                       load_dir=None,
                       lr=1e-4,
                       save_interval=10)
-
-    model = GLMForSeq2Seq.from_pretrain(model_name='GLM-large-ch')
-
+    model_name = 'GLM-large-ch'
+    tokenizer = Tokenizer.from_pretrained(model_name)
     ds_args = PretrainDatasetArguments()
-
-    tokenizer = GLMLargeChTokenizer()
-
     ds_args = add_args(ds_args, tokenizer)
+    model = GLMForSeq2Seq.from_pretrain(model_name=model_name)
 
     def create_dataset(tokenizer, should_split):
         dataset = get_dataset_lazy("./examples/glm_pretrain/data",
@@ -59,7 +56,7 @@ def create_dataset(tokenizer, should_split):
     collate_fn = None
     if ds_args.block_lm:
         collate_fn = ConstructBlockStrategy(
-            tokenizer, 512, eod_token=tokenizer.get_command('eos').Id)
+            tokenizer, 512, eod_token=tokenizer.get_command_id('eos'))
     metric_methods = DEFAULT_METRICS['pretrain']
     trainer.train(model,
                   collate_fn=collate_fn,
diff --git a/examples/glm_seq2seq/train.py b/examples/glm_seq2seq/train.py
index 39d3521f..81e5201f 100644
--- a/examples/glm_seq2seq/train.py
+++ b/examples/glm_seq2seq/train.py
@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSeq2Seq
-from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.data.dataset import Seq2SeqDataset
 from flagai.test_utils import Seq2SeqCollateArguments
 from flagai.data.dataset.superglue.control import DEFAULT_METRICS, CH_TASKS
@@ -27,12 +27,12 @@
 print("downloading...")
 
 if task_name in CH_TASKS:
-    tokenizer = GLMLargeChTokenizer()
     model_name = 'GLM-large-ch'
 else:
-    tokenizer = GLMLargeEnWordPieceTokenizer()
     model_name = 'GLM-large-en'
 
+tokenizer = Tokenizer.from_pretrained(model_name)
+
 train_dataset = Seq2SeqDataset(task_name=task_name,
                                data_dir='./datasets/',
                                dataset_type='train',
diff --git a/examples/glm_superglue/train_10b_clue.py b/examples/glm_superglue/train_10b_clue.py
index a1dd6241..1b5ffe6f 100644
--- a/examples/glm_superglue/train_10b_clue.py
+++ b/examples/glm_superglue/train_10b_clue.py
@@ -4,7 +4,7 @@
 import os
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -21,11 +21,12 @@
                   save_dir="./glm_superglue_en",
                   save_interval=1)
 
+model_name = "GLM-large-ch"
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
                                              model_name="GLM-large-ch")
 
 
-tokenizer =  GLMLargeChTokenizer()
+tokenizer = Tokenizer.from_pretrained("GLM-large-ch")
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',
diff --git a/examples/glm_superglue/train_10b_superglue.py b/examples/glm_superglue/train_10b_superglue.py
index 7fa485e6..4fa0207c 100644
--- a/examples/glm_superglue/train_10b_superglue.py
+++ b/examples/glm_superglue/train_10b_superglue.py
@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -28,11 +28,11 @@
                   # deepspeed_config='./deepspeed.json',
                   # training_script=__file__)
 
+model_name = "GLM-large-en"
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
+                                             model_name=model_name)
                                              
-tokenizer = GLMLargeEnWordPieceTokenizer()
-
+tokenizer = Tokenizer.from_pretrained(model_name)
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',
diff --git a/examples/glm_superglue/train_prefix.py b/examples/glm_superglue/train_prefix.py
index 4df44c42..99ac3a3a 100644
--- a/examples/glm_superglue/train_prefix.py
+++ b/examples/glm_superglue/train_prefix.py
@@ -2,13 +2,12 @@
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
-from flagai.model.glm_model import GLMForSingleTokenCloze, GLMForMultiTokenCloze, GLMForMultiTokenClozeFast, GLMForSequenceClassification
-from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer
+from flagai.model.glm_model import GLMForSequenceClassification
+from flagai.data.tokenizer import Tokenizer
 
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
 from flagai.data.dataset.superglue.control import DEFAULT_METRICS, MULTI_TOKEN_TASKS, CH_TASKS
-import unittest
 from flagai.data.dataset import ConstructSuperglueStrategy
 
 
@@ -32,13 +31,10 @@
 
 if task_name in CH_TASKS:
     model_name = 'GLM-large-ch'
-    tokenizer = GLMLargeChTokenizer(add_block_symbols=True,
-                                    add_task_mask=False,
-                                    add_decoder_mask=False,
-                                    fix_command_token=True)
+    add_block_symbols=True,
 else:
     model_name = 'GLM-large-en'
-    tokenizer = GLMLargeEnWordPieceTokenizer()
+tokenizer = Tokenizer.from_pretrained(model_name)
 
 model = GLMForSequenceClassification.from_pretrain(model_name=model_name, spell_length=2,
                                                     class_num=3, tune_prefix_layers=1)
diff --git a/examples/glm_superglue/train_qqp_deepspeed.py b/examples/glm_superglue/train_qqp_deepspeed.py
index 3f24cb07..a8629789 100644
--- a/examples/glm_superglue/train_qqp_deepspeed.py
+++ b/examples/glm_superglue/train_qqp_deepspeed.py
@@ -2,19 +2,20 @@
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
-from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.model.glm_model import GLMForSingleTokenCloze, GLMForMultiTokenCloze
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
+from flagai.data.dataset.superglue.control import DEFAULT_METRICS, MULTI_TOKEN_TASKS, CH_TASKS
 
-task_name = 'qqp'
+task_name = 'boolq'
 trainer = Trainer(env_type='deepspeed',
-                  epochs=10,
+                  epochs=1000,
                   batch_size=512,
                   eval_interval=100,
                   log_interval=10,
-                  save_interval = 1e5,
+                  save_interval=1e5,
                   gradient_accumulation_steps=5,
                   checkpoint_activations=True,
                   fp16=True,
@@ -22,18 +23,25 @@
                   weight_decay=0.1,
                   save_dir="./qqp",
                   master_ip='127.0.0.1',
-                  master_port=17887,
+                  master_port=17810,
                   num_nodes=1,
                   num_gpus=2,
                   hostfile='./hostfile',
                   deepspeed_config='./deepspeed.json',
                   training_script=__file__)
 
-model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
+model_name = "GLM-large-en"
+tokenizer = Tokenizer.from_pretrained(model_name)
 
+if task_name in MULTI_TOKEN_TASKS:
+    model = GLMForMultiTokenCloze.from_pretrain(
+        download_path="/mnt/test_10b_models", model_name=model_name)
+else:
+    model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
+                                                 model_name=model_name)
 
-tokenizer = GLMLargeEnWordPieceTokenizer()
+# model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
+#                                              model_name="GLM-large-en")
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',
diff --git a/examples/glm_superglue/train_qqp_pytorch.py b/examples/glm_superglue/train_qqp_pytorch.py
index 94f72b0c..f4ae40d1 100644
--- a/examples/glm_superglue/train_qqp_pytorch.py
+++ b/examples/glm_superglue/train_qqp_pytorch.py
@@ -4,8 +4,7 @@
 
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.model.bert_model import BertForClsClassifier
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -28,11 +27,12 @@
                   warm_up=0.1,
                   save_dir="./glm_large_qqp_pytorch")
 
+model_name = "GLM-large-en"
+tokenizer = Tokenizer.from_pretrained(model_name)
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
+                                             model_name=model_name)
+
 
-#tokenizer = GLM10bENBPETokenizer()
-tokenizer = GLMLargeEnWordPieceTokenizer()
 
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
diff --git a/examples/glm_superglue/train_qqp_pytorch_fp16.py b/examples/glm_superglue/train_qqp_pytorch_fp16.py
index 6d5dfc2e..676c2672 100644
--- a/examples/glm_superglue/train_qqp_pytorch_fp16.py
+++ b/examples/glm_superglue/train_qqp_pytorch_fp16.py
@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -24,9 +24,10 @@
                   warm_up=0.1,
                   save_dir="./glm_large_qqp_pytorch_fp16")
 
+model_name = "GLM-large-en"
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
-tokenizer = GLMLargeEnWordPieceTokenizer()
+                                             model_name=model_name)
+tokenizer = Tokenizer.from_pretrained(model_name)
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',
diff --git a/examples/glm_superglue/train_qqp_pytorchddp.py b/examples/glm_superglue/train_qqp_pytorchddp.py
index b422cda7..0070fe63 100644
--- a/examples/glm_superglue/train_qqp_pytorchddp.py
+++ b/examples/glm_superglue/train_qqp_pytorchddp.py
@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -29,11 +29,11 @@
                   hostfile='./hostfile',
                   training_script=__file__)
 
+model_name = "GLM-large-en"
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
+                                             model_name=model_name)
 
-#tokenizer = GLM10bENBPETokenizer()
-tokenizer = GLMLargeEnWordPieceTokenizer()
+tokenizer = Tokenizer.from_pretrained(model_name)
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',
diff --git a/examples/glm_title_generation/train.py b/examples/glm_title_generation/train.py
index 2dae9c5a..927cd9ac 100644
--- a/examples/glm_title_generation/train.py
+++ b/examples/glm_title_generation/train.py
@@ -134,7 +134,7 @@ def __call__(self, batch):
 
 sents_src, sents_tgt = read_file()
 my_collate_fn = GLMPoetryDynamicCollateFN(
-    pad_id=tokenizer.get_command('pad').Id)
+    pad_id=tokenizer.get_command_id('pad'))
 
 data_len = len(sents_tgt)
 train_size = int(data_len * 0.8)
diff --git a/examples/opt/generate_opt_1.3b.py b/examples/opt/generate_opt_1.3b.py
index fae238d2..8311a9f1 100644
--- a/examples/opt/generate_opt_1.3b.py
+++ b/examples/opt/generate_opt_1.3b.py
@@ -1,4 +1,3 @@
-
 from flagai.model.predictor.predictor import Predictor
 from flagai.auto_model.auto_loader import AutoLoader
 
diff --git a/examples/roberta_faq/1_construct_data.py b/examples/roberta_faq/1_construct_data.py
index d2671926..bcec3785 100644
--- a/examples/roberta_faq/1_construct_data.py
+++ b/examples/roberta_faq/1_construct_data.py
@@ -10,7 +10,6 @@
 import numpy as np
 from tqdm import tqdm
 import collections
-import faiss
 
 faq_data_path = "./data/financezhidao_filter.csv"
 answer_save_path = "./data/finance_fqa.json"
diff --git a/examples/swinv1/imagenet2012/train/n02939185/n02939185_57876.JPEG b/examples/swinv1/imagenet2012/train/n02939185/n02939185_57876.JPEG
new file mode 100644
index 00000000..fa5dd40d
Binary files /dev/null and b/examples/swinv1/imagenet2012/train/n02939185/n02939185_57876.JPEG differ
diff --git a/examples/swinv1/imagenet2012/val/n13044778/ILSVRC2012_val_00027938.JPEG b/examples/swinv1/imagenet2012/val/n13044778/ILSVRC2012_val_00027938.JPEG
new file mode 100644
index 00000000..571e3b3c
Binary files /dev/null and b/examples/swinv1/imagenet2012/val/n13044778/ILSVRC2012_val_00027938.JPEG differ
diff --git a/examples/swinv1/inference_swinv1.py b/examples/swinv1/inference_swinv1.py
new file mode 100644
index 00000000..0d843dd3
--- /dev/null
+++ b/examples/swinv1/inference_swinv1.py
@@ -0,0 +1,66 @@
+import os
+import torch
+import torchvision.datasets as datasets
+from flagai.auto_model.auto_loader import AutoLoader
+from torch.utils.data import DataLoader
+from torchvision import transforms
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+data_path = "./imagenet2012/"
+
+loader = AutoLoader(task_name="classification",
+                    model_name='swinv1-base-patch4-window7-224',
+                    num_classes=1000)
+model = loader.get_model()
+model.eval()
+model = model.to(device)
+
+def data_loader(root, batch_size=64, workers=8):
+    valdir = os.path.join(root, 'val')
+    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                     std=[0.229, 0.224, 0.225])
+
+    val_dataset = datasets.ImageFolder(
+        valdir,
+        transforms.Compose([
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            normalize
+        ])
+    )
+
+    val_loader = DataLoader(val_dataset,
+                            batch_size=batch_size,
+                            shuffle=False,
+                            num_workers=workers
+                            )
+    
+    return val_loader
+
+# 测试预训练权重
+@torch.no_grad()
+def test(model,data_loader):
+
+    model.eval()
+    top1_acc = 0.0
+
+    for step, (inputs, labels) in enumerate(data_loader):
+
+        inputs, labels = inputs.to(device), labels.to(device)
+        outputs = model(images=inputs)["logits"]
+
+        _, top1_preds = outputs.max(1)
+        top1_acc += top1_preds.eq(labels).sum().item()
+
+
+    print(
+            "test_top1_acc [{top1_acc}] \n".format(
+             top1_acc=top1_acc/len(data_loader.dataset),
+            )
+    )
+
+if __name__ == '__main__':
+
+    val_loader = data_loader(data_path)
+    test(model,val_loader)
diff --git a/examples/swinv1/training_swinv1.py b/examples/swinv1/training_swinv1.py
new file mode 100644
index 00000000..d19017da
--- /dev/null
+++ b/examples/swinv1/training_swinv1.py
@@ -0,0 +1,78 @@
+import os
+import torch
+import torchvision.datasets as datasets
+from flagai.auto_model.auto_loader import AutoLoader
+from torchvision import transforms
+from flagai.trainer import Trainer
+
+data_path = "./imagenet2012/"
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+trainer = Trainer(env_type="pytorch",
+                  epochs=10,
+                  experiment_name="swinv1_imagenet",
+                  batch_size=16,
+                  weight_decay=1e-3,
+                  warm_up=0.1,
+                  lr=5e-5,
+                  save_interval=1000,
+                  eval_interval=1000,
+                  log_interval=10,
+                  num_gpus=1)
+
+loader = AutoLoader(task_name="classification",
+                    model_name='swinv1-base-patch4-window7-224',
+                    num_classes=1000)
+model = loader.get_model()
+
+# build imagenet dataset
+def build_dataset(root):
+    traindir = os.path.join(root, 'train')
+    valdir = os.path.join(root, 'val')
+    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                     std=[0.229, 0.224, 0.225])
+
+    train_dataset = datasets.ImageFolder(
+        traindir,
+        transforms.Compose([
+            transforms.RandomResizedCrop(224),
+            transforms.RandomHorizontalFlip(),
+            transforms.ToTensor(),
+            normalize
+        ])
+    )
+    val_dataset = datasets.ImageFolder(
+        valdir,
+        transforms.Compose([
+            transforms.Resize((224, 224)),
+            transforms.ToTensor(),
+            normalize
+        ])
+    )
+    return train_dataset, val_dataset
+
+def collate_fn(batch):
+    images = torch.stack([b[0] for b in batch])
+    if trainer.fp16:
+        images = images.half()
+    labels = [b[1] for b in batch]
+    labels = torch.tensor(labels).long()
+    return {"images": images, "labels": labels}
+
+def top1_acc(pred, labels, **kwargs):
+    pred = pred.argmax(dim=1)
+    top1_acc = pred.eq(labels).sum().item() / len(pred)
+    return top1_acc
+
+if __name__ == '__main__':
+
+    print("building imagenet dataset......")
+    train_dataset, val_dataset = build_dataset(root=data_path)
+    print("training......")
+    optimizer = torch.optim.Adam(model.parameters(), lr=5e-5)
+    trainer.train(model,
+                  train_dataset=train_dataset,
+                  valid_dataset=val_dataset,
+                  collate_fn=collate_fn,
+                  metric_methods=[["top1_acc", top1_acc]],
+                  find_unused_parameters=False)
diff --git a/examples/swinv2/imagenet2012/train/n02939185/n02939185_57876.JPEG b/examples/swinv2/imagenet2012/train/n02939185/n02939185_57876.JPEG
new file mode 100644
index 00000000..fa5dd40d
Binary files /dev/null and b/examples/swinv2/imagenet2012/train/n02939185/n02939185_57876.JPEG differ
diff --git a/examples/swinv2/imagenet2012/val/n13044778/ILSVRC2012_val_00027938.JPEG b/examples/swinv2/imagenet2012/val/n13044778/ILSVRC2012_val_00027938.JPEG
new file mode 100644
index 00000000..571e3b3c
Binary files /dev/null and b/examples/swinv2/imagenet2012/val/n13044778/ILSVRC2012_val_00027938.JPEG differ
diff --git a/examples/swinv2/inference_swinv2.py b/examples/swinv2/inference_swinv2.py
new file mode 100644
index 00000000..56de7454
--- /dev/null
+++ b/examples/swinv2/inference_swinv2.py
@@ -0,0 +1,71 @@
+import torch
+import os
+from torchvision import transforms
+from torch.utils.data import Dataset, DataLoader
+import torchvision.datasets as datasets
+from tqdm import tqdm
+from flagai.auto_model.auto_loader import AutoLoader
+
+data_path = "./imagenet2012/"
+
+# swinv2 model_name support:
+# 1. swinv2-base-patch4-window16-256,
+# 2. swinv2-small-patch4-window16-256,
+# 3. swinv2-base-patch4-window8-256
+loader = AutoLoader(task_name="classification",
+                    model_name="swinv2-small-patch4-window16-256")
+model = loader.get_model()
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+model.eval()
+model.to(device)
+
+# imagenet loader
+def data_loader(root, batch_size=256, workers=1):
+    valdir = os.path.join(root, 'val')
+    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                     std=[0.229, 0.224, 0.225])
+
+    val_dataset = datasets.ImageFolder(
+        valdir,
+        transforms.Compose([
+            transforms.Resize((256, 256)),
+            transforms.ToTensor(),
+            normalize
+        ])
+    )
+
+    val_loader = DataLoader(val_dataset,
+                            batch_size=batch_size,
+                            shuffle=False,
+                            num_workers=workers
+                            )
+
+    return val_loader
+
+@torch.no_grad()
+def test(model,data_loader):
+    model.eval()
+    top1_acc = 0.0
+    top5_acc = 0.0
+
+    for step, (inputs, labels) in tqdm(enumerate(data_loader), total=len(data_loader)):
+        inputs, labels = inputs.to(device), labels.to(device)
+        outputs = model(inputs)["logits"]
+        _, top1_preds = outputs.max(1)
+        top1_acc += top1_preds.eq(labels).sum().item()
+
+        top5_pred = outputs.topk(5, 1, True)[1]
+        top5_acc += top5_pred.eq(labels.view(-1, 1).expand_as(top5_pred).to(device)).sum().item()
+
+    print(
+        "test_top1_acc [{top1_acc}], test_top5_acc [{top5_acc}] \n".format(
+            top1_acc=top1_acc/len(data_loader.dataset),
+            top5_acc=top5_acc/len(data_loader.dataset),
+        )
+    )
+if __name__ == '__main__':
+
+    val_loader = data_loader(data_path, batch_size=8, workers=8)
+    test(model, val_loader)
diff --git a/examples/swinv2/training_swinv2.py b/examples/swinv2/training_swinv2.py
new file mode 100644
index 00000000..5c8b0400
--- /dev/null
+++ b/examples/swinv2/training_swinv2.py
@@ -0,0 +1,87 @@
+import os
+import torch
+import torchvision.transforms as transforms
+import torchvision.datasets as datasets
+from flagai.trainer import Trainer
+from flagai.auto_model import AutoLoader
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+data_path = "./imagenet2012/"
+
+# use DDP for training by 4 gpus.
+trainer = Trainer(env_type="pytorchDDP",
+                  epochs=10,
+                  experiment_name="swinv2_imagenet_ddp",
+                  batch_size=32,
+                  weight_decay=1e-3,
+                  warm_up=0.1,
+                  lr=5e-5,
+                  save_interval=100,
+                  eval_interval=100,
+                  log_interval=10,
+                  num_gpus=4,
+                  hostfile="./hostfile",
+                  training_script="training_swinv2.py"
+                  )
+
+# swinv2 model_name support:
+# 1. swinv2-base-patch4-window16-256,
+# 2. swinv2-small-patch4-window16-256,
+# 3. swinv2-base-patch4-window8-256
+loader = AutoLoader(task_name="classification",
+                    model_name="swinv2-base-patch4-window8-256",
+                    num_classes=1000)
+model = loader.get_model()
+
+# build imagenet dataset
+def build_dataset(root):
+    traindir = os.path.join(root, 'train')
+    valdir = os.path.join(root, 'val')
+    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                     std=[0.229, 0.224, 0.225])
+
+    train_dataset = datasets.ImageFolder(
+        traindir,
+        transforms.Compose([
+            transforms.RandomResizedCrop(256),
+            transforms.RandomHorizontalFlip(),
+            transforms.ToTensor(),
+            normalize
+        ])
+    )
+    val_dataset = datasets.ImageFolder(
+        valdir,
+        transforms.Compose([
+            transforms.Resize((256, 256)),
+            transforms.ToTensor(),
+            normalize
+        ])
+    )
+    return train_dataset, val_dataset
+
+def collate_fn(batch):
+    images = torch.stack([b[0] for b in batch])
+    if trainer.fp16:
+        images = images.half()
+    labels = [b[1] for b in batch]
+    labels = torch.tensor(labels).long()
+    return {"images": images, "labels": labels}
+
+def top1_acc(pred, labels, **kwargs):
+    pred = pred.argmax(dim=1)
+    top1_acc = pred.eq(labels).sum().item() / len(pred)
+    return top1_acc
+
+if __name__ == '__main__':
+
+    print("building imagenet dataset......")
+    train_dataset, val_dataset = build_dataset(root=data_path)
+    print("training......")
+    optimizer = torch.optim.Adam(model.parameters(), lr=5e-5)
+    trainer.train(model,
+                  train_dataset=train_dataset,
+                  valid_dataset=val_dataset,
+                  collate_fn=collate_fn,
+                  optimizer=optimizer,
+                  metric_methods=[["top1_acc", top1_acc]],
+                  find_unused_parameters=False)
\ No newline at end of file
diff --git a/examples/t5_flagai_11b/train_title_with_flagai_t5_11b.py b/examples/t5_flagai_11b/train_title_with_flagai_t5_11b.py
index 83912033..c052b284 100644
--- a/examples/t5_flagai_11b/train_title_with_flagai_t5_11b.py
+++ b/examples/t5_flagai_11b/train_title_with_flagai_t5_11b.py
@@ -4,6 +4,7 @@
 from flagai.trainer import Trainer
 from flagai.model.t5_model import T5ForConditionalGeneration
 from transformers import T5Tokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.model.predictor.predictor import Predictor
 from torch.utils.data import Dataset
 import os
@@ -53,7 +54,8 @@ def read_file():
 
     return src, tgt
 
-tokenizer = T5Tokenizer.from_pretrained('t5-11b')
+# t5-11b is not uploaded to modelhub yet. Since it shares tokenizer with T5-base-en, we will get tokenizer here
+tokenizer = Tokenizer.from_pretrained('T5-base-en')
  # path to your downloaded model files is /mnt/t5-11b
 model = T5ForConditionalGeneration.from_pretrain(download_path='/mnt',
                                                  model_name='t5-11b',checkpoint_activations=True)
diff --git a/flagai/auto_model/__init__.py b/flagai/auto_model/__init__.py
index e69de29b..1baa1d8b 100644
--- a/flagai/auto_model/__init__.py
+++ b/flagai/auto_model/__init__.py
@@ -0,0 +1,2 @@
+
+from .auto_loader import *
\ No newline at end of file
diff --git a/flagai/auto_model/auto_loader.py b/flagai/auto_model/auto_loader.py
index c805f794..46bae3b6 100644
--- a/flagai/auto_model/auto_loader.py
+++ b/flagai/auto_model/auto_loader.py
@@ -7,7 +7,6 @@
 from  flagai.model.file_utils import _get_model_id, _get_vocab_path
 import copy
 
-
 class LazyImport(object):
 
     def __init__(self, name):
@@ -21,25 +20,25 @@ def __getattr__(self, name):
             self.cache[self.mod_name] = mod
         return getattr(mod, name)
 
-
+# 2 columns : 1-package name,  2-class name
 ALL_TASK = {
     "bert_lm": ["flagai.model.bert_model", "BertModel"],
     "bert_seq2seq": ["flagai.model.bert_model", "BertForSeq2seq"],
     "bert_title-generation": ["flagai.model.bert_model", "BertForSeq2seq"],
     "bert_masklm": ["flagai.model.bert_model", "BertForMaskLM"],
     "bert_sequence-labeling":
-    ["flagai.model.bert_model", "BertForSequenceLabeling"],
+        ["flagai.model.bert_model", "BertForSequenceLabeling"],
     "bert_sequence-labeling-crf":
-    ["flagai.model.bert_model", "BertForSequenceLabeling"],
+        ["flagai.model.bert_model", "BertForSequenceLabeling"],
     "bert_sequence-labeling-gp":
-    ["flagai.model.bert_model", "BertForSequenceLabeling"],
+        ["flagai.model.bert_model", "BertForSequenceLabeling"],
     "bert_ner": ["flagai.model.bert_model", "BertForSequenceLabeling"],
     "bert_ner-crf": ["flagai.model.bert_model", "BertForSequenceLabelingCRF"],
     "bert_ner-gp": ["flagai.model.bert_model", "BertForSequenceLabelingGP"],
     "bert_embedding": ["flagai.model.bert_model", "BertForEmbedding"],
     "bert_classification": ["flagai.model.bert_model", "BertForClsClassifier"],
     "bert_semantic-matching":
-    ["flagai.model.bert_model", "BertForClsClassifier"],
+        ["flagai.model.bert_model", "BertForClsClassifier"],
     "gpt2_seq2seq": ("flagai.model.gpt2_model", "GPT2Model"),
     "gpt2_lm": ("flagai.model.gpt2_model", "GPT2Model"),
     "cpm_seq2seq": ("flagai.model.cpm_model", "CPMModel"),
@@ -50,14 +49,17 @@ def __getattr__(self, name):
     "glm_seq2seq": ["flagai.model.glm_model", "GLMForSeq2Seq"],
     "glm_poetry": ["flagai.model.glm_model", "GLMForSeq2Seq"],
     "glm_classification":
-    ["flagai.model.glm_model", "GLMForSequenceClassification"],
+        ["flagai.model.glm_model", "GLMForSequenceClassification"],
     "glm_title-generation": ["flagai.model.glm_model", "GLMForSeq2Seq"],
     "opt_seq2seq": ("flagai.model.opt_model","OPTModel"),
     "opt_lm": ("flagai.model.opt_model","OPTModel"),
-    "vit_classification": ("flagai.model.vision.vit", "VisionTransformer")
-
+    "vit_classification": ("flagai.model.vision.vit", "VisionTransformer"),
+    "clip_txt_img_matching": ("flagai.model.mm.clip_model", "CLIP"),
+    "swinv1_classification": ("flagai.model.vision.swinv1", "SwinTransformer"),
+    "swinv2_classification": ("flagai.model.vision.swinv2", "SwinTransformerV2"),
 }
 
+# 4 columns : 1-package name,  2-class name, 3-model brief name, 4-model type
 MODEL_DICT = {
     "bert-base-en": ["flagai.model.bert_model", "BertModel", "bert", "nlp"],
     "roberta-base-ch": ["flagai.model.bert_model", "BertModel", "bert", "nlp"],
@@ -66,7 +68,7 @@ def __getattr__(self, name):
     "glm-large-ch": ["flagai.model.glm_model", "GLMModel", "glm", "nlp"],
     "glm-large-en": ["flagai.model.glm_model", "GLMModel", "glm", "nlp"],
     "gpt2-base-ch": ["flagai.model.gpt2_model", "GPT2Model", "gpt2", "nlp"],
-    "cpm-large-ch": ["flagai.model.cpm_model", "CPMModel", "cpm", "nlp"],
+    "cpm-large-ch": ["flagai.model.gpt2_model", "GPT2Model", "cpm", "nlp"],
     "opt-125m-en": ["flagai.model.opt_model","OPTModel", "opt", "nlp"],
     "opt-350m-en": ["flagai.model.opt_model","OPTModel", "opt", "nlp"],
     "opt-1.3b-en": ["flagai.model.opt_model","OPTModel", "opt", "nlp"],
@@ -85,37 +87,19 @@ def __getattr__(self, name):
     "vit-large-p16-384":["flagai.model.vision.vit", "VisionTransformer", "vit", "vision"],
     "vit-large-p32-224":["flagai.model.vision.vit", "VisionTransformer", "vit", "vision"],
     "vit-large-p32-384":["flagai.model.vision.vit", "VisionTransformer", "vit", "vision"],
-}
+    "clip-base-p32-224":["flagai.model.mm.clip_model", "CLIP", "clip", "mm"],
+    "clip-base-p16-224":["flagai.model.mm.clip_model", "CLIP", "clip", "mm"],
+    "clip-large-p14-224":["flagai.model.mm.clip_model", "CLIP", "clip", "mm"],
+    "clip-large-p14-336":["flagai.model.mm.clip_model", "CLIP", "clip", "mm"],
+
+    "swinv1-base-patch4-window7-224": ["flagai.model.vision.swinv1", "SwinTransformer","swinv1","vision"],
 
-TOKENIZER_DICT = {
-    "bert-base-en": ["flagai.data.tokenizer.bert.bert_tokenizer", "BertTokenizer"],
-    "roberta-base-ch": ["flagai.data.tokenizer.bert.bert_tokenizer", "BertTokenizer"],
-    "t5-base-en": ["flagai.data.tokenizer.t5.t5_pegasus_tokenizer", "T5PegasusTokenizer"],
-    "t5-base-ch": ["flagai.data.tokenizer.t5.t5_pegasus_tokenizer", "T5PegasusTokenizer"],
-    "glm-large-ch": [
-        "flagai.data.tokenizer.glm_large_ch.glm_large_ch_tokenizer",
-        "GLMLargeChTokenizer"
-    ],
-    "glm-large-en": [
-        "flagai.data.tokenizer.glm_large_en.glm_large_en_tokenizer",
-        "GLMLargeEnWordPieceTokenizer"
-    ],
-    "glm-10b-ch": [
-        "flagai.data.tokenizer.glm_large_ch.glm_large_ch_tokenizer",
-        "GLMLargeChTokenizer"
-    ],
-    "gpt2-base-ch": ["flagai.data.tokenizer.bert.bert_tokenizer", "BertTokenizer"],
-    "cpm-large-ch": ["flagai.data.tokenizer.cpm_1.cpm1_tokenizer", "CPMTokenizer"],
-    "opt-125m-en": ["flagai.data.tokenizer.opt.opt_en_tokenizer","OPTTokenizer"],
-    "opt-350m-en": ["flagai.data.tokenizer.opt.opt_en_tokenizer","OPTTokenizer"],
-    "opt-1.3b-en": ["flagai.data.tokenizer.opt.opt_en_tokenizer","OPTTokenizer"],
-    "opt-2.7b-en": ["flagai.data.tokenizer.opt.opt_en_tokenizer","OPTTokenizer"],
-    "opt-6.7b-en": ["flagai.data.tokenizer.opt.opt_en_tokenizer","OPTTokenizer"],
-    "opt-13b-en": ["flagai.data.tokenizer.opt.opt_en_tokenizer","OPTTokenizer"],
-    "opt-30b-en": ["flagai.data.tokenizer.opt.opt_en_tokenizer","OPTTokenizer"],
-    "opt-66b-en": ["flagai.data.tokenizer.opt.opt_en_tokenizer","OPTTokenizer"],
+    "swinv2-base-patch4-window8-256": ["flagai.model.vision.swinv2", "SwinTransformerV2", "swinv2", "vision"],
+    "swinv2-base-patch4-window16-256": ["flagai.model.vision.swinv2", "SwinTransformerV2", "swinv2", "vision"],
+    "swinv2-small-patch4-window16-256": ["flagai.model.vision.swinv2", "SwinTransformerV2", "swinv2", "vision"],
 }
 
+
 class AutoLoader:
 
     def __init__(self,
@@ -151,7 +135,7 @@ def __init__(self,
                                          class_num=2)
 
         """
-        
+
         raw_model_name = copy.deepcopy(model_name)
 
         model_name = model_name.lower()
@@ -176,7 +160,6 @@ def __init__(self,
             )
             return
 
-
         model_id = _get_model_id(f"{raw_model_name}-{task_name}")
         if model_id != 'null':
             model_name_ = f"{raw_model_name}-{task_name}"
@@ -186,49 +169,24 @@ def __init__(self,
         os.makedirs(download_path, exist_ok=True)
         self.model = getattr(LazyImport(self.model_name[0]),
                              self.model_name[1]).from_pretrain(
-                                 download_path=model_dir,
-                                 model_name=model_name_,
-                                 only_download_config=only_download_config,
-                                 device=device,
-                                 **kwargs)
-
-        model_id = _get_model_id(model_name)
+            download_path=model_dir,
+            model_name=model_name_,
+            only_download_config=only_download_config,
+            device=device,
+            **kwargs)
+
+        try:
+            model_id = _get_model_id(model_name)
+        except:
+            print("Model hub is not reachable!")
+            model_id = -1
 
         print("*"*20, task_name, model_id, model_name)
-        if model_type == "nlp":
-            if "glm" in model_name and "ch" in model_name:
-                vocab_file = os.path.join(download_path,'cog-pretrained.model')
-                if not os.path.exists(vocab_file):
-                    vocab_file = _get_vocab_path(download_path, "cog-pretrain.model", model_id)
-            elif "glm" in model_name and "en" in model_name:
-                vocab_file = "GLM-large-en"
-            elif model_name == "cpm-large-ch":
-                # two files to load
-                vocab_file_1 = os.path.join(download_path, "vocab.json")
-                vocab_file_2 = os.path.join(download_path, "chinese_vocab.model")
-                if not os.path.exists(vocab_file_1):
-                    vocab_file_1 = _get_vocab_path(download_path, "vocab.json",
-                                                   model_id)
-                if not os.path.exists(vocab_file_2):
-                    vocab_file_2 = _get_vocab_path(download_path,
-                                                   "chinese_vocab.model", model_id)
-            else:
-                vocab_file = os.path.join(download_path, 'vocab.txt')
-                if not os.path.exists(vocab_file):
-                    vocab_file = _get_vocab_path(download_path, "vocab.txt",
-                                                 model_id)
-            tokenizer_class = TOKENIZER_DICT[model_name]
-            tokenizer_class = getattr(LazyImport(tokenizer_class[0]),
-                                        tokenizer_class[1])
-            if model_name == "cpm-large-ch":
-                self.tokenizer = tokenizer_class(vocab_file_1, vocab_file_2)
-            elif brief_model_name == "opt":
-                self.tokenizer = tokenizer_class("facebook/opt-350m")
-            elif model_name in ["glm-large-en", "glm-large-ch"]:
-                self.tokenizer = tokenizer_class()
-            else :
-                self.tokenizer = tokenizer_class(vocab_file)
-        elif model_type == "vision":
+        if model_type == "mm" or model_type == "nlp":
+            tokenizer_class = getattr(LazyImport("flagai.data.tokenizer"),
+                                      "Tokenizer")
+            self.tokenizer = tokenizer_class.from_pretrained(model_name)
+        else :
             self.tokenizer = None
 
     def get_task_name(self, brief_model_name):
@@ -246,5 +204,4 @@ def get_model(self):
 
     def load_pretrain_params(self, model_path):
         self.model.load_huggingface_weights(model_path)
-
         print(f"Loading done: {model_path}")
diff --git a/flagai/data/dataset/block/blocklm_utils.py b/flagai/data/dataset/block/blocklm_utils.py
index 5f773ef0..4687305f 100644
--- a/flagai/data/dataset/block/blocklm_utils.py
+++ b/flagai/data/dataset/block/blocklm_utils.py
@@ -87,11 +87,11 @@ def __init__(self,
         self.shuffle_blocks = shuffle_blocks
         self.sentinel_token = sentinel_token
         self.generation_mask = 'gMASK' if task_mask else 'MASK'
-        self.generation_mask = self.tokenizer.get_command(
-            self.generation_mask).Id
+        self.generation_mask = self.tokenizer.get_command_id(
+            self.generation_mask)
         self.gap_sentence_mask = 'sMASK' if task_mask else 'MASK'
-        self.gap_sentence_mask = self.tokenizer.get_command(
-            self.gap_sentence_mask).Id
+        self.gap_sentence_mask = self.tokenizer.get_command_id(
+            self.gap_sentence_mask)
         self.random_position = random_position
         self.masked_lm = masked_lm
 
@@ -145,7 +145,7 @@ def sample_span_in_document(self, tokens, masked_lengths, rng):
         for index in reversed(indices):
             start_index = index
             if start_index + 1 < len(tokens) and tokens[
-                    start_index + 1] == self.tokenizer.get_command('ENC').Id:
+                    start_index + 1] == self.tokenizer.get_command_id('cls'):
                 start_index += 1
             length = last_index - start_index - 1
             if last_index == len(tokens) and length > 0:
@@ -205,7 +205,7 @@ def make_masked_data(self,
         #
         position_ids = np.arange(len(tokens), dtype=np.int64)
         targets = copy.deepcopy(tokens)
-        mask_id = self.tokenizer.get_command('MASK').Id
+        mask_id = self.tokenizer.get_command_id('MASK')
         mlm_masks = np.zeros(len(tokens), dtype=np.int64)
         for start, end in block_spans:
             for idx in range(start, end):
@@ -242,16 +242,16 @@ def make_block_data(self,
         target_tokens, target_position_ids, target_block_position_ids, targets = [], [], [], []
         for start, end, idx in block_spans:
             sop_token = 'sop' if idx == 0 else f"sop{idx}"
-            target_tokens.append([self.tokenizer.get_command(sop_token).Id])
+            target_tokens.append([self.tokenizer.get_command_id(sop_token)])
             span_tokens = copy.deepcopy(tokens[start:end])
             if self.block_mask_prob > 0.0 and task == 'bert':
                 for sub_idx in range(len(span_tokens)):
                     if random.random() < self.block_mask_prob:
-                        span_tokens[sub_idx] = self.tokenizer.get_command(
-                            'dBLOCK').Id
+                        span_tokens[sub_idx] = self.tokenizer.get_command_id(
+                            'dBLOCK')
             target_tokens.append(span_tokens)
             targets.append(tokens[start:end])
-            targets.append([self.tokenizer.get_command('eop').Id])
+            targets.append([self.tokenizer.get_command_id('eop')])
             if not self.sentinel_token:
                 target_position_id = position_ids[start:end]
                 target_position_ids.append(target_position_id)
@@ -274,7 +274,7 @@ def make_block_data(self,
                 mask_id = self.gap_sentence_mask
             else:
                 mask_token = 'MASK' if idx == 0 else f'MASK{idx}'
-                mask_id = self.tokenizer.get_command(mask_token).Id
+                mask_id = self.tokenizer.get_command_id(mask_token)
             local_spans.append((current_length, current_length + start - last))
             source_tokens.append(tokens[last:start])
             source_tokens.append([mask_id])
@@ -296,7 +296,7 @@ def make_block_data(self,
             raise RuntimeError
         if self.encoder_decoder:
             target_tokens = target_tokens + [
-                self.tokenizer.get_command('eop').Id
+                self.tokenizer.get_command_id('eop')
             ]
             loss_masks = np.ones(len(target_tokens), dtype=np.int64)
             return source_tokens, target_tokens, loss_masks
@@ -315,7 +315,7 @@ def make_block_data(self,
                     mask_candidates,
                     int(self.context_mask_ratio * text_length))
                 for pos in mask_pos:
-                    tokens[pos] = self.tokenizer.get_command('dBLOCK').Id
+                    tokens[pos] = self.tokenizer.get_command_id('dBLOCK')
             targets = np.concatenate(source_tokens + targets)
             loss_masks = np.ones(len(tokens), dtype=np.int64)
             loss_masks[:source_length] = 0
@@ -338,7 +338,7 @@ def generate_blank_data(self,
                             task='bert'):
         rng.shuffle(masked_lengths)
         tokens, loss_masks = sample['text'], sample['loss_mask']
-        assert tokens[0] == self.tokenizer.get_command('ENC').Id
+        assert tokens[0] == self.tokenizer.get_command_id('cls')
         block_spans = self.sample_span_in_document(tokens, masked_lengths, rng)
         if len(block_spans) < len(masked_lengths):
             return None
@@ -358,8 +358,8 @@ def split_samples(self, samples, rng):
         target_length = rng.randrange(32, self.max_seq_length - 1)
         num_splits = (self.max_seq_length - 1) // target_length
         new_samples = []
-        cls_id = self.tokenizer.get_command('ENC').Id
-        eos_id = self.tokenizer.get_command('eos').Id
+        cls_id = self.tokenizer.get_command_id('cls')
+        eos_id = self.tokenizer.get_command_id('eos')
         for sample in samples:
             tokens, loss_masks = sample['text'][1:], sample['loss_mask'][1:]
             for _ in range(num_splits):
@@ -458,14 +458,14 @@ def __call__(self, samples):
             for sample in samples:
                 tokens, loss_masks = sample['text'], sample['loss_mask']
                 sentence_spans = []
-                last_index = 1 if tokens[0] == self.tokenizer.get_command(
-                    'ENC').Id else 0
+                last_index = 1 if tokens[0] == self.tokenizer.get_command_id(
+                    'cls') else 0
                 for i in range(len(tokens)):
                     if self.contains_sentence_end(tokens[i]):
                         if last_index < i + 1:
                             sentence_spans.append((last_index, i + 1))
                         last_index = i + 1
-                    elif tokens[i] == self.tokenizer.get_command('eos').Id:
+                    elif tokens[i] == self.tokenizer.get_command_id('eos'):
                         last_index = i + 1
                 if last_index < len(tokens):
                     sentence_spans.append((last_index, len(tokens)))
@@ -507,7 +507,7 @@ def __call__(self, samples):
                     len(sample['text']) - generation_length + 1)
                 multiple_doc = index_in_list(
                     sample['text'],
-                    self.tokenizer.get_command('eos').Id) not in [
+                    self.tokenizer.get_command_id('eos')) not in [
                         -1, len(sample['text']) - 1
                     ]
                 if multiple_doc or rng.random() < self.infill_prob:
@@ -518,7 +518,7 @@ def __call__(self, samples):
                     target_masks = loss_masks[division:]
                     tokens = np.concatenate((source_tokens, [
                         self.generation_mask,
-                        self.tokenizer.get_command('sop').Id
+                        self.tokenizer.get_command_id('sop')
                     ], target_tokens[:-1]))
                     targets = np.concatenate(
                         (source_tokens, [self.generation_mask], target_tokens))
diff --git a/flagai/data/dataset/block/dataset.py b/flagai/data/dataset/block/dataset.py
index cdd79af9..afbb43c2 100644
--- a/flagai/data/dataset/block/dataset.py
+++ b/flagai/data/dataset/block/dataset.py
@@ -112,22 +112,22 @@ def __getitem__(self, idx):
                                 tokens[strip_left_tokens - 1]):
                         strip_left_tokens += 1
                         move_count += 1
-            tokens = [self.tokenizer.get_command('ENC').Id
+            tokens = [self.tokenizer.get_command_id('cls')
                       ] + tokens[strip_left_tokens:]
             loss_mask = [0] + loss_mask[strip_left_tokens:]
-            if len(tokens) == 2 and tokens[1] == self.tokenizer.get_command(
-                    'eos').Id:
+            if len(tokens) == 2 and tokens[1] == self.tokenizer.get_command_id(
+                    'eos'):
                 tokens, loss_mask = [], []
             tokens, loss_mask = self.right_strip_seq(tokens, loss_mask,
                                                      self.max_seq_len)
         else:
-            tokens = [self.tokenizer.get_command('ENC').Id] + tokens
+            tokens = [self.tokenizer.get_command_id('cls')] + tokens
             loss_mask = [0] + loss_mask
             # Sample multiple documents
             if self.sample_across_doc:
                 while len(tokens) < self.max_seq_len:
                     new_tokens, new_loss_mask = self.get_weighted_samples(rng)
-                    new_tokens = [self.tokenizer.get_command('ENC').Id
+                    new_tokens = [self.tokenizer.get_command_id('cls')
                                   ] + new_tokens
                     new_loss_mask = [0] + new_loss_mask
                     is_last = len(new_tokens) >= self.max_seq_len - len(tokens)
@@ -159,7 +159,7 @@ def right_strip_seq(self, tokens, loss_mask, seq_length):
     def getidx(self, data_idx):
         data = self.ds[data_idx]
         tokens, loss_masks = data['tokens'], data['loss_masks']
-        tokens = tokens + [self.tokenizer.get_command('eos').Id]
+        tokens = tokens + [self.tokenizer.get_command_id('eos')]
         loss_masks = loss_masks + [1]
         return tokens, loss_masks
 
@@ -167,7 +167,7 @@ def pad_seq(self, seq, pad_id=None):
         total_tokens = self.max_seq_len
         num_pad_tokens = max(0, total_tokens - len(seq))
         seq += [
-            self.tokenizer.get_command('pad').Id if pad_id is None else pad_id
+            self.tokenizer.get_command_id('pad') if pad_id is None else pad_id
         ] * (num_pad_tokens)
         return seq
 
diff --git a/flagai/data/dataset/data_collator/collate_fn.py b/flagai/data/dataset/data_collator/collate_fn.py
index ed3037f0..f162d1f2 100644
--- a/flagai/data/dataset/data_collator/collate_fn.py
+++ b/flagai/data/dataset/data_collator/collate_fn.py
@@ -121,12 +121,12 @@ def __init__(self, args, tokenizer, task_name):
         self.args = args
 
     def encode(self, example):
-        cls_id = self.tokenizer.get_command('ENC').Id
+        cls_id = self.tokenizer.get_command_id('cls')
         mask_token = 'sMASK' if self.args.task_mask else 'MASK'
-        mask_id = self.tokenizer.get_command(mask_token).Id
-        pad_id = self.tokenizer.get_command('pad').Id
-        sop_id = self.tokenizer.get_command('sop').Id
-        eop_id = self.tokenizer.get_command('eop').Id
+        mask_id = self.tokenizer.get_command_id(mask_token)
+        pad_id = self.tokenizer.get_command_id('pad')
+        sop_id = self.tokenizer.get_command_id('sop')
+        eop_id = self.tokenizer.get_command_id('eop')
         if self.task_name in [
                 "gigaword", "cnn_dm", "cnn_dm_original", "xsum", "lang8_hsk"
         ]:
@@ -171,7 +171,7 @@ def sub_finder(mylist, pattern):
             source_tokens = [cls_id] + source_tokens + [mask_id
                                                         ] + answer_tokens
         elif self.task_name in ["cmrc"]:
-            mask_id = self.tokenizer.get_command('MASK').Id
+            mask_id = self.tokenizer.get_command_id('MASK')
             source_text = example.text_a
             target_text = example.meta["answer"].strip()
             question = example.meta["question"].strip()
@@ -187,7 +187,7 @@ def sub_finder(mylist, pattern):
                 mask_id
             ] + source_tokens[:max_src_length]
         elif self.task_name in ["wsc"]:
-            mask_id = self.tokenizer.get_command('MASK').Id
+            mask_id = self.tokenizer.get_command_id('MASK')
             source_text = example.text_a
             target_text = example.meta["answer"].strip()
             question = example.meta["question"].strip()
@@ -304,11 +304,11 @@ def __init__(self,
         self.shuffle_blocks = shuffle_blocks
         self.sentinel_token = sentinel_token
         self.generation_mask = 'gMASK' if task_mask else 'MASK'
-        self.generation_mask = self.tokenizer.get_command(
-            self.generation_mask).Id
+        self.generation_mask = self.tokenizer.get_command_id(
+            self.generation_mask)
         self.gap_sentence_mask = 'sMASK' if task_mask else 'MASK'
-        self.gap_sentence_mask = self.tokenizer.get_command(
-            self.gap_sentence_mask).Id
+        self.gap_sentence_mask = self.tokenizer.get_command_id(
+            self.gap_sentence_mask)
         self.random_position = random_position
         self.masked_lm = masked_lm
 
@@ -362,7 +362,7 @@ def sample_span_in_document(self, tokens, masked_lengths, rng):
         for index in reversed(indices):
             start_index = index
             if start_index + 1 < len(tokens) and tokens[
-                    start_index + 1] == self.tokenizer.get_command('ENC').Id:
+                    start_index + 1] == self.tokenizer.get_command_id('cls'):
                 start_index += 1
             length = last_index - start_index - 1
             if last_index == len(tokens) and length > 0:
@@ -422,7 +422,7 @@ def make_masked_data(self,
 
         position_ids = np.arange(len(tokens), dtype=np.int64)
         targets = copy.deepcopy(tokens)
-        mask_id = self.tokenizer.get_command('MASK').Id
+        mask_id = self.tokenizer.get_command_id('MASK')
         mlm_masks = np.zeros(len(tokens), dtype=np.int64)
         for start, end in block_spans:
             for idx in range(start, end):
@@ -459,16 +459,16 @@ def make_block_data(self,
         target_tokens, target_position_ids, target_block_position_ids, targets = [], [], [], []
         for start, end, idx in block_spans:
             sop_token = 'sop' if idx == 0 else f"sop{idx}"
-            target_tokens.append([self.tokenizer.get_command(sop_token).Id])
+            target_tokens.append([self.tokenizer.get_command_id(sop_token)])
             span_tokens = copy.deepcopy(tokens[start:end])
             if self.block_mask_prob > 0.0 and task == 'bert':
                 for sub_idx in range(len(span_tokens)):
                     if random.random() < self.block_mask_prob:
-                        span_tokens[sub_idx] = self.tokenizer.get_command(
-                            'dBLOCK').Id
+                        span_tokens[sub_idx] = self.tokenizer.get_command_id(
+                            'dBLOCK')
             target_tokens.append(span_tokens)
             targets.append(tokens[start:end])
-            targets.append([self.tokenizer.get_command('eop').Id])
+            targets.append([self.tokenizer.get_command_id('eop')])
             if not self.sentinel_token:
                 target_position_id = position_ids[start:end]
                 target_position_ids.append(target_position_id)
@@ -491,7 +491,7 @@ def make_block_data(self,
                 mask_id = self.gap_sentence_mask
             else:
                 mask_token = 'MASK' if idx == 0 else f'MASK{idx}'
-                mask_id = self.tokenizer.get_command(mask_token).Id
+                mask_id = self.tokenizer.get_command_id(mask_token)
             local_spans.append((current_length, current_length + start - last))
             source_tokens.append(tokens[last:start])
             source_tokens.append([mask_id])
@@ -513,7 +513,7 @@ def make_block_data(self,
             raise RuntimeError
         if self.encoder_decoder:
             target_tokens = target_tokens + [
-                self.tokenizer.get_command('eop').Id
+                self.tokenizer.get_command_id('eop')
             ]
             loss_masks = np.ones(len(target_tokens), dtype=np.int64)
             return source_tokens, target_tokens, loss_masks
@@ -532,7 +532,7 @@ def make_block_data(self,
                     mask_candidates,
                     int(self.context_mask_ratio * text_length))
                 for pos in mask_pos:
-                    tokens[pos] = self.tokenizer.get_command('dBLOCK').Id
+                    tokens[pos] = self.tokenizer.get_command_id('dBLOCK')
             targets = np.concatenate(source_tokens + targets)
             loss_masks = np.ones(len(tokens), dtype=np.int64)
             loss_masks[:source_length] = 0
@@ -555,7 +555,7 @@ def generate_blank_data(self,
                             task='bert'):
         rng.shuffle(masked_lengths)
         tokens, loss_masks = sample['input_ids'], sample['loss_mask']
-        assert tokens[0] == self.tokenizer.get_command('ENC').Id
+        assert tokens[0] == self.tokenizer.get_command_id('cls')
         block_spans = self.sample_span_in_document(tokens, masked_lengths, rng)
         if len(block_spans) < len(masked_lengths):
             return None
@@ -575,8 +575,8 @@ def split_samples(self, samples, rng):
         target_length = rng.randrange(32, self.max_seq_length - 1)
         num_splits = (self.max_seq_length - 1) // target_length
         new_samples = []
-        cls_id = self.tokenizer.get_command('ENC').Id
-        eos_id = self.tokenizer.get_command('eos').Id
+        cls_id = self.tokenizer.get_command_id('cls')
+        eos_id = self.tokenizer.get_command_id('eos')
         for sample in samples:
             tokens, loss_masks = sample['input_ids'][1:], sample['loss_mask'][
                 1:]
@@ -676,14 +676,14 @@ def __call__(self, samples):
             for sample in samples:
                 tokens, loss_masks = sample['input_ids'], sample['loss_mask']
                 sentence_spans = []
-                last_index = 1 if tokens[0] == self.tokenizer.get_command(
-                    'ENC').Id else 0
+                last_index = 1 if tokens[0] == self.tokenizer.get_command_id(
+                    'cls') else 0
                 for i in range(len(tokens)):
                     if self.contains_sentence_end(tokens[i]):
                         if last_index < i + 1:
                             sentence_spans.append((last_index, i + 1))
                         last_index = i + 1
-                    elif tokens[i] == self.tokenizer.get_command('eos').Id:
+                    elif tokens[i] == self.tokenizer.get_command_id('eos'):
                         last_index = i + 1
                 if last_index < len(tokens):
                     sentence_spans.append((last_index, len(tokens)))
@@ -725,7 +725,7 @@ def __call__(self, samples):
                     len(sample['input_ids']) - generation_length + 1)
                 multiple_doc = index_in_list(
                     sample['input_ids'],
-                    self.tokenizer.get_command('eos').Id) not in [
+                    self.tokenizer.get_command_id('eos')) not in [
                         -1, len(sample['input_ids']) - 1
                     ]
                 if multiple_doc or rng.random() < self.infill_prob:
@@ -737,7 +737,7 @@ def __call__(self, samples):
                     target_masks = loss_masks[division:]
                     tokens = np.concatenate((source_tokens, [
                         self.generation_mask,
-                        self.tokenizer.get_command('sop').Id
+                        self.tokenizer.get_command_id('sop')
                     ], target_tokens[:-1]))
                     targets = np.concatenate(
                         (source_tokens, [self.generation_mask], target_tokens))
diff --git a/flagai/data/dataset/data_utils.py b/flagai/data/dataset/data_utils.py
index 98f65720..4f0ee38d 100644
--- a/flagai/data/dataset/data_utils.py
+++ b/flagai/data/dataset/data_utils.py
@@ -134,10 +134,10 @@ def build_input_from_ids(text_a_ids,
 
     # Prepare ids for special tokens
     if mask_id is None:
-        mask_id = tokenizer.get_command('MASK').Id
-    eos_id = tokenizer.get_command('eos').Id  # end of sentence token
-    cls_id = tokenizer.get_command('ENC').Id  # start of sentence token
-    sep_id = tokenizer.get_command('sep').Id  # seperator of two texts token
+        mask_id = tokenizer.get_command_id('MASK')
+    eos_id = tokenizer.get_command_id('eos')  # end of sentence token
+    cls_id = tokenizer.get_command_id('cls')  # start of sentence token
+    sep_id = tokenizer.get_command_id('sep')  # seperator of two texts token
 
     ids = []  # ids of all the tokens
     types = [
@@ -191,7 +191,7 @@ def build_input_from_ids(text_a_ids,
     block_position_ids = [0] * len(ids)
     # Piece
     if add_piece or answer_ids is not None:
-        sop_id = tokenizer.get_command('sop').Id
+        sop_id = tokenizer.get_command_id('sop')
         mask_position = ids.index(
             mask_id
         ) if not args.sentinel_token else args.max_position_embeddings
@@ -235,9 +235,9 @@ def build_input_from_ids(text_a_ids,
 #
 def build_decoder_input(enc_ids, answer_ids, max_seq_length,
                         max_dec_seq_length, tokenizer):
-    mask_id = tokenizer.get_command('MASK').Id
-    eos_id = tokenizer.get_command('eos').Id
-    sop_id = tokenizer.get_command('sop').Id
+    mask_id = tokenizer.get_command_id('MASK')
+    eos_id = tokenizer.get_command_id('eos')
+    sop_id = tokenizer.get_command_id('sop')
     masks = []
     # TODO: it probably takes too much memory
     # for i in range(max_dec_seq_length):
diff --git a/flagai/data/dataset/language_model/dataset.py b/flagai/data/dataset/language_model/dataset.py
index 318761e4..b291251b 100644
--- a/flagai/data/dataset/language_model/dataset.py
+++ b/flagai/data/dataset/language_model/dataset.py
@@ -39,7 +39,7 @@ def __init__(self, args, documents, tokenizer, num_original_tokens,
         self.unidirectional = args.unidirectional
         self.block_lm = args.block_lm
         mask_token = "gMASK" if args.task_mask else 'MASK'
-        self.mask_id = self.tokenizer.get_command(mask_token).Id
+        self.mask_id = self.tokenizer.get_command_id(mask_token)
 
     def __len__(self):
         return sum(self.num_sequences)
@@ -111,12 +111,12 @@ def __init__(self, args, tokenizer, strict=True):
         self.args = args
         self.max_seq_length = args.seq_length
         self.tokenizer = tokenizer
-        self.pad_idx = tokenizer.get_command('pad').Id
+        self.pad_idx = tokenizer.get_command_id('pad')
         self.strict = strict
         self.block_lm = args.block_lm
         self.unidirectional = args.unidirectional
         mask_token = "gMASK" if args.task_mask else 'MASK'
-        self.mask_id = self.tokenizer.get_command(mask_token).Id
+        self.mask_id = self.tokenizer.get_command_id(mask_token)
 
         self.tokens = []
         self.labels = []
diff --git a/flagai/data/dataset/mm/__init__.py b/flagai/data/dataset/mm/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/flagai/data/dataset/mm/clip_dataset.py b/flagai/data/dataset/mm/clip_dataset.py
new file mode 100644
index 00000000..a05eaf4d
--- /dev/null
+++ b/flagai/data/dataset/mm/clip_dataset.py
@@ -0,0 +1,57 @@
+import torch
+from PIL import Image
+import os
+from torch.utils.data import Dataset
+import pandas as pd
+from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
+
+def _convert_image_to_rgb(image):
+    return image.convert("RGB")
+
+def clip_transform(img_size):
+    transform = Compose([
+    Resize(img_size, interpolation=Image.BICUBIC),
+    CenterCrop(img_size),
+    _convert_image_to_rgb,
+    ToTensor(),
+    Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
+    ])
+  
+    return transform
+
+class CsvDataset(Dataset):
+    def __init__(self, 
+                input_filename,
+                img_dir,
+                transforms, 
+                tokenizer, 
+                img_key="filepath", 
+                caption_key="title", 
+                sep="\t"):
+        print(f'Loading csv data from {input_filename}.')
+        df = pd.read_csv(input_filename, sep=sep)
+        self.img_dir = img_dir
+        self.img_names = df[img_key].tolist()
+        self.captions = df[caption_key].tolist()
+        self.transforms = transforms
+        self.tokenizer = tokenizer
+        print('Done loading data.')
+
+    def __len__(self):
+        return len(self.captions)
+
+    def __getitem__(self, idx):
+        image = Image.open(os.path.join(self.img_dir, self.img_names[idx]))
+        images = self.transforms(image)
+        texts = self.tokenizer.tokenize_as_tensor([str(self.captions[idx])])[0]
+        return images, texts
+
+def collate_fn(batch):
+    images = [b[0] for b in batch]
+    texts = [b[1] for b in batch]
+    images = torch.stack(images, dim=0).float()
+    texts = torch.stack(texts, dim=0).long()
+    return {
+        "image": images,
+        "text": texts
+    }
\ No newline at end of file
diff --git a/flagai/data/dataset/seq2seq/dataset.py b/flagai/data/dataset/seq2seq/dataset.py
index af4e02dd..1313141c 100644
--- a/flagai/data/dataset/seq2seq/dataset.py
+++ b/flagai/data/dataset/seq2seq/dataset.py
@@ -388,7 +388,7 @@ def __init__(self,
                  src_seq_length=608,
                  tgt_seq_length=160):
         self.task_name = task_name
-        self.data_dir = data_dir + task_name + '/'
+        self.data_dir = data_dir 
         self.max_src_length, self.max_tgt_length = src_seq_length, tgt_seq_length
         self.dataset_type = dataset_type
         self.tokenizer = tokenizer
@@ -403,13 +403,13 @@ def __init__(self,
             self.processor = SummmaryProcessor(self.task_name, self.data_dir,
                                                tokenizer)
         elif self.task_name in ["xsum"]:
-            self.processor = XSumProcessor(self.data_dir, tokenizer)
+            self.processor = XSumProcessor(self.data_dir+task_name, tokenizer)
         elif self.task_name in ["squad_generation"]:
-            self.processor = SQuADProcessor(self.data_dir, tokenizer)
+            self.processor = SQuADProcessor(self.data_dir+task_name, tokenizer)
         elif self.task_name in ['cmrc']:
-            self.processor = CMRCProcessor(self.data_dir, tokenizer)
+            self.processor = CMRCProcessor(self.data_dir+task_name, tokenizer)
         elif self.task_name in ['wsc']:
-            self.processor = WSCProcessor(self.data_dir, tokenizer)
+            self.processor = WSCProcessor(self.data_dir+task_name, tokenizer)
         else:
             raise NotImplementedError(self.task_name)
         example_list = self.processor.create_examples(dataset_type)
@@ -425,132 +425,6 @@ def __len__(self):
     def __getitem__(self, idx):
         example = self.example_list[idx]
         return example
-        # cls_id = self.tokenizer.get_command('ENC').Id
-        # mask_token = 'sMASK' if self.task_mask else 'MASK'
-        # mask_id = self.tokenizer.get_command(mask_token).Id
-        # pad_id = self.tokenizer.get_command('pad').Id
-        # sop_id = self.tokenizer.get_command('sop').Id
-        # eop_id = self.tokenizer.get_command('eop').Id
-        # if self.task_name in [
-        #         "gigaword", "cnn_dm", "cnn_dm_original", "xsum", "lang8_hsk"
-        # ]:
-        #     source_text, target_text = example.text_a, example.text_b
-        #     source_tokens = self.tokenizer.EncodeAsIds(" " + source_text)
-        #     prompt = [cls_id, mask_id
-        #               ] + self.tokenizer.EncodeAsIds(" Content:")
-        #     if len(source_tokens) > self.max_src_length - len(prompt):
-        #         source_tokens = source_tokens[:self.max_src_length -
-        #                                       len(prompt)]
-        #     source_tokens = prompt + source_tokens
-        # elif self.task_name == "squad_generation":
-        #     source_text = example.text_a
-        #     target_text, answer = example.meta["question"], example.meta[
-        #         "answer"]
-        #     source_tokens = self.tokenizer.EncodeAsIds(source_text.rstrip() +
-        #                                                " Question:")
-        #     answer_tokens = self.tokenizer.EncodeAsIds(" Answer: " + answer)
-        #     if len(source_tokens
-        #            ) > self.max_src_length - len(answer_tokens) - 2:
-        #         max_src_length = self.max_src_length - len(answer_tokens) - 2
-        #         answer_pattern = self.tokenizer.EncodeAsIds(" " + answer)
-        #
-        #         def sub_finder(mylist, pattern):
-        #             matches = []
-        #             for i in range(len(mylist)):
-        #                 if mylist[i] == pattern[0] and mylist[
-        #                         i:i + len(pattern)] == pattern:
-        #                     matches.append(i)
-        #             return matches
-        #
-        #         answer_indices = sub_finder(source_tokens, answer_pattern)
-        #         if len(answer_indices) == 0:
-        #             print(f"Answer {answer} not exists in the source text")
-        #             source_tokens = source_tokens[:max_src_length]
-        #         else:
-        #             start_index = max(answer_indices[0] - max_src_length // 2,
-        #                               0)
-        #             source_tokens = source_tokens[start_index:start_index +
-        #                                           max_src_length]
-        #     source_tokens = [cls_id] + source_tokens + [mask_id
-        #                                                 ] + answer_tokens
-        # elif self.task_name in ["cmrc"]:
-        #     mask_id = self.tokenizer.get_command('MASK').Id
-        #     source_text = example.text_a
-        #     target_text = example.meta["answer"].strip()
-        #     question = example.meta["question"].strip()
-        #     source_tokens = self.tokenizer.EncodeAsIds(source_text.rstrip())
-        #     question_tokens = self.tokenizer.EncodeAsIds("问题：" + question +
-        #                                                  "答案：")
-        #     max_src_length = self.max_src_length - len(question_tokens) - 2
-        #     if max_src_length <= 0:
-        #         question_tokens = question_tokens[self.max_src_length // 4]
-        #     source_tokens = [cls_id] + question_tokens + [
-        #         mask_id
-        #     ] + source_tokens[:max_src_length]
-        # elif self.task_name in ["wsc"]:
-        #     mask_id = self.tokenizer.get_command('MASK').Id
-        #     source_text = example.text_a
-        #     target_text = example.meta["answer"].strip()
-        #     question = example.meta["question"].strip()
-        #     source_tokens = self.tokenizer.EncodeAsIds(source_text.rstrip())
-        #     question_tokens = self.tokenizer.EncodeAsIds("what does " +
-        #                                                  question + "mean: ")
-        #     max_src_length = self.max_src_length - len(question_tokens) - 2
-        #     if max_src_length <= 0:
-        #         print(question)
-        #         question_tokens = question_tokens[self.max_src_length // 4]
-        #     source_tokens = [cls_id] + question_tokens + [
-        #         mask_id
-        #     ] + source_tokens[:max_src_length]
-        # else:
-        #     raise NotImplementedError
-        # if len(source_tokens) < self.max_src_length:
-        #     source_tokens = source_tokens + [pad_id] * (self.max_src_length -
-        #                                                 len(source_tokens))
-        # sep = len(source_tokens)
-        # position_ids = list(range(len(source_tokens)))
-        # block_position_ids = [0] * len(source_tokens)
-        # mask_pos = source_tokens.index(mask_id)
-        # if self.dataset_type == 'train' or self.dataset_type == "dev":
-        #     target_tokens = self.tokenizer.EncodeAsIds(" " + target_text)
-        #     target_tokens = target_tokens + [eop_id]
-        #     if len(target_tokens) > self.max_tgt_length:
-        #         target_tokens = target_tokens[:self.max_tgt_length]
-        #     loss_mask = [1] * len(target_tokens)
-        #     if len(target_tokens) < self.max_tgt_length:
-        #         loss_mask += [0] * (self.max_tgt_length - len(target_tokens))
-        #         target_tokens += [pad_id] * (self.max_tgt_length -
-        #                                      len(target_tokens))
-        #     tokens = source_tokens + [sop_id] + target_tokens[:-1]
-        #     loss_mask = [0] * len(source_tokens) + loss_mask
-        #     target_ids = [0] * len(source_tokens) + target_tokens
-        #     position_ids += [mask_pos] * len(target_tokens)
-        #     if self.no_block_position:
-        #         block_position_ids += [1] * len(target_tokens)
-        #     else:
-        #         block_position_ids += list(range(1, len(target_tokens) + 1))
-        #     position_ids = [position_ids, block_position_ids]
-        #     sample = {
-        #         'input_ids': np.array(tokens, dtype=np.int64),
-        #         'target_ids': np.array(target_ids, dtype=np.int64),
-        #         'attention_mask': np.array(sep, dtype=np.int64),
-        #         'loss_mask': np.array(loss_mask, dtype=np.int64),
-        #         "position_ids": np.array(position_ids, dtype=np.int64),
-        #         "uid": example.guid
-        #     }
-        # else:
-        #     tokens = source_tokens + [sop_id]
-        #     position_ids = position_ids + [mask_pos]
-        #     block_position_ids = block_position_ids + [1]
-        #     position_ids = [position_ids, block_position_ids]
-        #     sample = {
-        #         'input_ids': np.array(tokens, dtype=np.int64),
-        #         'attention_mask': np.array(sep, dtype=np.int64),
-        #         "position_ids": np.array(position_ids, dtype=np.int64),
-        #         "uid": example.guid
-        #     }
-        # return sample
-
 
 class ExtractionDataset(torch.utils.data.Dataset):
 
@@ -604,10 +478,10 @@ def __getitem__(self, idx):
         example = self.example_list[idx]
         source_text, target_text = example.text_a, example.text_b
         mask_token = 'MASK'
-        mask_id = self.tokenizer.get_command(mask_token).Id
-        sop_id = self.tokenizer.get_command('sop').Id
-        eop_id = self.tokenizer.get_command('eop').Id
-        pad_id = self.tokenizer.get_command('pad').Id
+        mask_id = self.tokenizer.get_command_id(mask_token)
+        sop_id = self.tokenizer.get_command_id('sop')
+        eop_id = self.tokenizer.get_command_id('eop')
+        pad_id = self.tokenizer.get_command_id('pad')
 
         def pad_to(text, max_len, pad_id):
             if len(text) > max_len:
@@ -739,10 +613,10 @@ def __getitem__(self, idx):
         example = self.example_list[idx]
         source_text = example.text_a
         mask_token = 'gMASK' if self.args.task_mask else 'MASK'
-        mask_id = self.tokenizer.get_command(mask_token).Id
-        sop_id = self.tokenizer.get_command('sop').Id
-        eop_id = self.tokenizer.get_command('eop').Id
-        pad_id = self.tokenizer.get_command('pad').Id
+        mask_id = self.tokenizer.get_command_id(mask_token)
+        sop_id = self.tokenizer.get_command_id('sop')
+        eop_id = self.tokenizer.get_command_id('eop')
+        pad_id = self.tokenizer.get_command_id('pad')
         if self.split in ['train', 'dev']:
             masked_src, masked_tgt = self.mask_text(source_text)
             source_text = masked_src
diff --git a/flagai/data/dataset/superglue/control.py b/flagai/data/dataset/superglue/control.py
index 2baf5061..2f63899e 100644
--- a/flagai/data/dataset/superglue/control.py
+++ b/flagai/data/dataset/superglue/control.py
@@ -170,8 +170,11 @@ def _download_data(self, dirname, dname):
         files = [f for f in os.listdir(dirname)]
 
         for f in files:
-            if f.lower() == dname:
-                os.rename(dirname + '/' + f, dirname + '/' + dname)
+            try:
+                if f.lower() == dname:
+                    os.rename(dirname + '/' + f, dirname + '/' + dname)
+            except:
+                pass
 
     def _unzip_file(self, src_file, dst_dir):
         r = zipfile.is_zipfile((src_file))
diff --git a/flagai/data/dataset/superglue/pvp.py b/flagai/data/dataset/superglue/pvp.py
index 631a2545..d6e6cce6 100644
--- a/flagai/data/dataset/superglue/pvp.py
+++ b/flagai/data/dataset/superglue/pvp.py
@@ -97,12 +97,12 @@ def spell_length(self):
     @property
     def mask(self) -> str:
         """Return the underlying LM's mask token"""
-        return self.tokenizer.get_command('MASK').Id
+        return self.tokenizer.get_command_id('MASK')
 
     @property
     def mask_id(self) -> int:
         """Return the underlying LM's mask id"""
-        return self.tokenizer.get_command('MASK').Id
+        return self.tokenizer.get_command_id('MASK')
 
     @property
     def max_num_verbalizers(self) -> int:
@@ -264,7 +264,7 @@ def encode_input(raw_parts):
                         answer_ids = get_verbalization_ids(
                             answer, tokenizer, force_single_token=False)
                         answer_ids = answer_ids + [
-                            tokenizer.get_command('eop').Id
+                            tokenizer.get_command_id('eop')
                         ]
                         self.num_truncated += self.truncate(
                             this_parts_a,
@@ -376,7 +376,7 @@ def encode_input(raw_parts):
                 for answer in answers:
                     answer_ids = get_verbalization_ids(
                         answer, tokenizer, force_single_token=False)
-                    answer_ids = answer_ids + [tokenizer.get_command('eop').Id]
+                    answer_ids = answer_ids + [tokenizer.get_command_id('eop')]
                     answer_ids = answer_ids[:self.max_dec_seq_length]
                     data = build_decoder_input(ids, answer_ids,
                                                self.max_seq_length,
@@ -510,7 +510,6 @@ def get_answers(self, example: InputExample):
     def get_verbalizer_ids(self):
         target_ids = []
         for label in self.label_list:
-
             verbalizer = self.verbalize(label)[0]
             verbalizer_id = get_verbalization_ids(verbalizer,
                                                   self.tokenizer,
@@ -575,13 +574,13 @@ def spell_length(self):
     def mask(self) -> str:
         """Return the underlying LM's mask token"""
         mask_token = 'MASK'
-        return self.tokenizer.get_command(mask_token).Id
+        return self.tokenizer.get_command_id(mask_token)
 
     @property
     def mask_id(self) -> int:
         """Return the underlying LM's mask id"""
         mask_token = 'MASK'
-        return self.tokenizer.get_command(mask_token).Id
+        return self.tokenizer.get_command_id(mask_token)
 
     def get_answers(self, example: InputExample):
         choice1 = " " + self.remove_final_punc(
@@ -658,7 +657,7 @@ def encode(self,
             get_verbalization_ids(answer, tokenizer, force_single_token=True)
         ]
         if self.is_multi_token:
-            answer_ids.append(tokenizer.get_command('eop').Id)
+            answer_ids.append(tokenizer.get_command_id('eop'))
 
         ids_list, positions_list, sep_list, mask_list, target_list = [], [], [], [], []
 
@@ -814,7 +813,7 @@ def encode_input(raw_parts):
         answer_ids = get_verbalization_ids(answer,
                                            tokenizer,
                                            force_single_token=False)
-        answer_ids = answer_ids + [tokenizer.get_command('eop').Id]
+        answer_ids = answer_ids + [tokenizer.get_command_id('eop')]
         self.num_truncated += self.truncate(parts_a,
                                             parts_b,
                                             answer_ids,
@@ -1690,7 +1689,6 @@ def get_verbalization_ids(word: str, tokenizer,
     :return: either the list of token ids or the single token id corresponding to this word
     """
     if force_single_token:
-        # verbalization_id = tokenizer.TokenToId(word)
         verbalization_id = tokenizer.TokenToId(word)
         assert verbalization_id not in tokenizer.command_id_map, \
             f'Verbalization {word} is mapped to a special token {tokenizer.IdToToken(verbalization_id)}'
diff --git a/flagai/data/tokenizer/__init__.py b/flagai/data/tokenizer/__init__.py
index c33872d7..e07653af 100644
--- a/flagai/data/tokenizer/__init__.py
+++ b/flagai/data/tokenizer/__init__.py
@@ -5,4 +5,6 @@
 from .roberta.roberta_tokenizer import ROBERTATokenizer
 from .bert.bert_tokenizer import BertWordPieceTokenizer
 from .cpm_1.cpm1_tokenizer import CPMTokenizer
-from .opt.opt_en_tokenizer import OPTTokenizer
\ No newline at end of file
+from .opt.opt_en_tokenizer import OPTTokenizer
+from .uni_tokenizer.tokenizer import Tokenizer
+# from .uni_tokenizer.base_tokenizer import BaseTokenizer
diff --git a/flagai/data/tokenizer/clip/__init__.py b/flagai/data/tokenizer/clip/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/flagai/data/tokenizer/clip/tokenizer.py b/flagai/data/tokenizer/clip/tokenizer.py
new file mode 100644
index 00000000..b1c5d830
--- /dev/null
+++ b/flagai/data/tokenizer/clip/tokenizer.py
@@ -0,0 +1,175 @@
+""" CLIP tokenizer
+
+Copied from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI.
+"""
+import gzip
+import html
+import os
+from functools import lru_cache
+from typing import Union, List
+
+import ftfy
+import regex as re
+import torch
+
+
+@lru_cache()
+def default_bpe():
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+
+def whitespace_clean(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+class ClipTokenizer(object):
+    def __init__(self, bpe_path: str = default_bpe(), special_tokens=None):
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        merges = open(bpe_path).read().split('\n')
+        merges = merges[1:49152-256-2+1]
+        merges = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v+'</w>' for v in vocab]
+        for merge in merges:
+            vocab.append(''.join(merge))
+        if not special_tokens:
+            special_tokens = ['<start_of_text>', '<end_of_text>']
+        else:
+            special_tokens = ['<start_of_text>', '<end_of_text>'] + special_tokens
+        vocab.extend(special_tokens)
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {t:t for t in special_tokens}
+        special = "|".join(special_tokens)
+        self.pat = re.compile(special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
+
+        self.vocab_size = len(self.encoder)
+        self.all_special_ids = [self.encoder[t] for t in special_tokens]
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + ( token[-1] + '</w>',)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token+'</w>'
+
+        while True:
+            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word)-1 and word[i+1] == second:
+                    new_word.append(first+second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
+        return text
+
+    def tokenize(self, texts: Union[str, List[str]], context_length: int = 77) -> torch.LongTensor:
+        """
+        Returns the tokenized representation of given input string(s)
+
+        Parameters
+        ----------
+        texts : Union[str, List[str]]
+            An input string or a list of input strings to tokenize
+        context_length : int
+            The context length to use; all CLIP models use 77 as the context length
+
+        Returns
+        -------
+        A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
+        """
+        if isinstance(texts, str):
+            texts = [texts]
+
+        sot_token = self.encoder["<start_of_text>"]
+        eot_token = self.encoder["<end_of_text>"]
+        all_tokens = [[sot_token] + self.encode(text) + [eot_token] for text in texts]
+        result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+        for i, tokens in enumerate(all_tokens):
+            if len(tokens) > context_length:
+                tokens = tokens[:context_length]  # Truncate
+            result[i, :len(tokens)] = torch.tensor(tokens)
+        return result
diff --git a/flagai/data/tokenizer/glm_10b_en/glm_10b_en_bpe_tokenizer.py b/flagai/data/tokenizer/glm_10b_en/glm_10b_en_bpe_tokenizer.py
index f7dc6dad..b762b66b 100644
--- a/flagai/data/tokenizer/glm_10b_en/glm_10b_en_bpe_tokenizer.py
+++ b/flagai/data/tokenizer/glm_10b_en/glm_10b_en_bpe_tokenizer.py
@@ -58,7 +58,7 @@ def __init__(self,
                              self.text_tokenizer.encoder['
']),
                 CommandToken('sep', '[SEP]',
                              self.text_tokenizer.encoder['
']),
-                CommandToken('ENC', '[CLS]',
+                CommandToken('cls', '[CLS]',
                              self.text_tokenizer.encoder['
']),
                 CommandToken('MASK',
                              '[MASK]',
@@ -87,7 +87,7 @@ def __init__(self,
                 self._command_tokens.extend([
                     CommandToken('sop', '
', self.num_tokens),
                     CommandToken('eop', '
', self.num_tokens + 1),
-                    CommandToken('ENC', '[CLS]', self.num_tokens + 2),
+                    CommandToken('cls', '[CLS]', self.num_tokens + 2),
                     CommandToken('MASK',
                                  '[MASK]',
                                  self.num_tokens + 3,
diff --git a/flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py b/flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py
index 062b0d8f..a7c2a281 100644
--- a/flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py
+++ b/flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py
@@ -127,29 +127,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
             else:
                 logger.info("loading special tokens file {}".format(
                     special_tokens_file))
-        # redirect to the cache, if necessary
-        # try:
-        #     resolved_vocab_file = cached_path(vocab_file, cache_dir=cache_dir)
-        #     resolved_merges_file = cached_path(merges_file, cache_dir=cache_dir)
-        # except EnvironmentError:
-        #     logger.error(
-        #         "Model name '{}' was not found in model name list ({}). "
-        #         "We assumed '{}' was a path or url but couldn't find files {} and {} "
-        #         "at this path or url.".format(
-        #             pretrained_model_name_or_path,
-        #             ', '.join(PRETRAINED_VOCAB_ARCHIVE_MAP.keys()),
-        #             pretrained_model_name_or_path,
-        #             vocab_file, merges_file))
-        #     return None
-        # if resolved_vocab_file == vocab_file and resolved_merges_file == merges_file:
-        #     logger.info("loading vocabulary file {}".format(vocab_file))
-        #     logger.info("loading merges file {}".format(merges_file))
-        # else:
-        #     logger.info("loading vocabulary file {} from cache at {}".format(
-        #         vocab_file, resolved_vocab_file))
-        #     logger.info("loading merges file {} from cache at {}".format(
-        #         merges_file, resolved_merges_file))
-        # print(os.getcwd())
+
         resolved_vocab_file = os.path.join(os.path.dirname(__file__),
                                            vocab_file)
         resolved_merges_file = os.path.join(os.path.dirname(__file__),
@@ -170,7 +148,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
         else:
             special_tokens = kwargs.pop('special_tokens', [])
 
-        if not os.path.exists(resolved_vocab_file):  # 如果没有这个vocab文件， 那我们就要下载
+        if not os.path.exists(resolved_merges_file):  
             if pretrained_model_name_or_path in VOCAB_ARCHIVE_URLS_MAP:
                 for key, url in VOCAB_ARCHIVE_URLS_MAP[
                         pretrained_model_name_or_path].items():
diff --git a/flagai/data/tokenizer/glm_large_ch/glm_large_ch.py b/flagai/data/tokenizer/glm_large_ch/glm_large_ch.py
index e7be92d2..00b1ec3f 100644
--- a/flagai/data/tokenizer/glm_large_ch/glm_large_ch.py
+++ b/flagai/data/tokenizer/glm_large_ch/glm_large_ch.py
@@ -35,7 +35,6 @@ def get_pairs(word):
 
 
 class Encoder:
-
     def __init__(self, encoder, bpe_merges):
         self.encoder = encoder
         self.decoder = {v: k for k, v in self.encoder.items()}
@@ -160,13 +159,8 @@ def get_encoder(encoder_file, bpe_file):
             bpe_merges=bpe_merges,
         )
 
-
 def from_pretrained(pretrained_model_file=None):
-    vocab_file = 'cog-pretrain.vocab'
-    model_file = 'cog-pretrain.model'
     if pretrained_model_file is None:
-        cache_dir = os.path.join(os.path.dirname(__file__), 'vocabs')
         model_id = _get_model_id("GLM-large-ch")
-        _get_vocab_path(cache_dir, vocab_file, model_id, rank=0)
-        _get_vocab_path(cache_dir, model_file, model_id, rank=0)
+        _get_vocab_path(pretrained_model_file, model_id, rank=0)
     return get_encoder(pretrained_model_file, "")
diff --git a/flagai/data/tokenizer/glm_large_ch/glm_large_ch_tokenizer.py b/flagai/data/tokenizer/glm_large_ch/glm_large_ch_tokenizer.py
index ea9d14e9..69048d3a 100644
--- a/flagai/data/tokenizer/glm_large_ch/glm_large_ch_tokenizer.py
+++ b/flagai/data/tokenizer/glm_large_ch/glm_large_ch_tokenizer.py
@@ -54,7 +54,7 @@ def __init__(self,
             CommandToken('pad', '
', self.num_text_tokens),
             CommandToken('eos', '
', self.num_text_tokens),
             CommandToken('sep', '[SEP]', self.num_text_tokens + 1),
-            CommandToken('ENC', '[CLS]', self.num_text_tokens + 2),
+            CommandToken('cls', '[CLS]', self.num_text_tokens + 2),
             CommandToken('MASK',
                          '[MASK]',
                          self.num_text_tokens + 3,
@@ -144,9 +144,9 @@ def encode_plus(  #for Seq2seq
             target_text=None,
     ):
 
-        sop_id = self.get_command('sop').Id  #start of piece
-        eop_id = self.get_command('eop').Id  #end of piece
-        sep_id = self.get_command('sep').Id  #seperation
+        sop_id = self.get_command_id('sop')  #start of piece
+        eop_id = self.get_command_id('eop')  #end of piece
+        sep_id = self.get_command_id('sep')  #seperation
 
         source_tokens = self.EncodeAsIds(source_text)
         source_tokens = [sop_id] + source_tokens + [sep_id]
@@ -199,9 +199,6 @@ def MultiWordId(self, exception=None):
         return result
 
     def CommandTokenIds(self, exception=None):
-        #get command tokens' ids
-        #return ids list
-        #exception token: string list
         result = []
         for s in self._command_tokens:
             if not exception or (exception and s.name not in exception):
@@ -214,10 +211,6 @@ def EncodeAsTokens(self, text, process_fn=None):
             processed_text = process_fn(processed_text)
         tokens = self.text_tokenizer.tokenize(processed_text)
         return tokens
-        # tokenization = Tokenization(tokens, processed_text, text, asIds=False)
-        # tokenization.set_command_tokens(self._command_tokens)
-        # return tokenization
-        # return Tokenization(tokens, processed_text, text, asIds=False)
 
     def IdToToken(self, Id, type_token=False):
         if isinstance(Id, (TypeToken, CommandToken)):
@@ -263,7 +256,5 @@ def DecodeTokens(self, Tokens, type_token=False):
         if type_token:
             return ' '.join(t.token if isinstance(t, TypeToken) else t
                             for t in Tokens)
-        # if isinstance(Tokens, Tokenization):
-        #     Tokens = Tokens.tokenization
         return self.text_tokenizer.decode(
             [self.TokenToId(tok) for tok in Tokens])
diff --git a/flagai/data/tokenizer/glm_large_en/glm_large_en_tokenizer.py b/flagai/data/tokenizer/glm_large_en/glm_large_en_tokenizer.py
index c981e869..ff4e1e4a 100644
--- a/flagai/data/tokenizer/glm_large_en/glm_large_en_tokenizer.py
+++ b/flagai/data/tokenizer/glm_large_en/glm_large_en_tokenizer.py
@@ -58,7 +58,7 @@ def __init__(self,
 
         self._command_tokens = [
             CommandToken('pad', '[PAD]', self.text_tokenizer.vocab['[PAD]']),
-            CommandToken('ENC', '[CLS]', self.text_tokenizer.vocab['[CLS]']),
+            CommandToken('cls', '[CLS]', self.text_tokenizer.vocab['[CLS]']),
             CommandToken('MASK', '[MASK]',
                          self.text_tokenizer.vocab['[MASK]']),
             CommandToken('unk', '[UNK]', self.text_tokenizer.vocab['[UNK]']),
diff --git a/flagai/data/tokenizer/glm_large_en/wordpiece.py b/flagai/data/tokenizer/glm_large_en/wordpiece.py
index 73521347..83338bcf 100644
--- a/flagai/data/tokenizer/glm_large_en/wordpiece.py
+++ b/flagai/data/tokenizer/glm_large_en/wordpiece.py
@@ -138,7 +138,7 @@ def from_pretrained(cls,
 
         model_id = _get_model_id(pretrained_model_name_or_path)
         if not os.path.exists(cache_dir + '/' +
-                              vocab_file):  # Temporary if statement
+                              vocab_file):
             _get_vocab_path(cache_dir + '/', vocab_file, model_id, rank=0)
 
         resolved_vocab_file = os.path.join(cache_dir, vocab_file)
diff --git a/flagai/data/tokenizer/tokenizer.py b/flagai/data/tokenizer/tokenizer.py
index da0fa494..3f82e7f5 100644
--- a/flagai/data/tokenizer/tokenizer.py
+++ b/flagai/data/tokenizer/tokenizer.py
@@ -53,7 +53,7 @@ def __str__(self):
     ('unk', 3),
     ('sep', 4),
     ('L2R', 5),
-    ('ENC', 6),
+    ('cls', 6),
     ('MASK', 7),
 ]
 DEFAULT_COMMAND_TOKENS = prep_command_tokens(DEFAULT_COMMAND_TOKENS)
@@ -96,6 +96,9 @@ def __str__(self):
 DEFAULT_TYPE_TOKENS = prep_type_tokens(DEFAULT_TYPE_TOKENS)
 
 
+
+
+
 class GLMTokenizer(object):
     """
     Tokenizer object that handles text tokenization, command tokens, and type tokens.
@@ -170,7 +173,7 @@ def __len__(self):
         """total number of tokens"""
         return self.num_tokens
 
-    def get_command(self, name):
+    def get_command_id(self, name):
         """get command token corresponding to `name`"""
         return self.command_name_map[name]
 
@@ -284,9 +287,6 @@ def split_on_tokens(tok_list, text):
         no_split_tokens = self._command_tokens
         Ids = split_on_tokens(no_split_tokens, processed_text)
         return Ids
-        # tokenization = Tokenization(Ids, processed_text, text)
-        # tokenization.set_command_tokens(self._command_tokens)
-        # return tokenization
 
     def _encode(self, text):
         raise NotImplementedError
@@ -370,6 +370,8 @@ def DecodeTokens(self, Tokens, type_token=False):
         return ' '.join(rtn_strs)
 
 
+
+
 class Tokenizer(object):
     """
     Tokenizer object that handles text tokenization, command tokens, and type tokens.
@@ -456,6 +458,8 @@ def DecodeTokens(self, tokens):
         return self.text_tokenizer.convert_tokens_to_string(tokens)
 
 
+# class BaseTokenizer(object):
+
 class TextTokenizer(object):
     """
     Interface for text tokenizer
diff --git a/flagai/data/tokenizer/uni_tokenizer/__init__.py b/flagai/data/tokenizer/uni_tokenizer/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/flagai/data/tokenizer/uni_tokenizer/base_tokenizer.py b/flagai/data/tokenizer/uni_tokenizer/base_tokenizer.py
new file mode 100644
index 00000000..08f2ba91
--- /dev/null
+++ b/flagai/data/tokenizer/uni_tokenizer/base_tokenizer.py
@@ -0,0 +1,90 @@
+import os
+from flagai.model.file_utils import _get_model_files, _get_model_id, _get_vocab_path
+from flagai.data.tokenizer.uni_tokenizer.properties import VOCAB_FILE, MERGES_FILE, SP_MODEL_FILE, VOCAB_JSON_FILE
+import warnings
+
+
+class BaseTokenizer(object):
+    @classmethod
+    def from_pretrained(cls,
+                        tokenizer_model_name,
+                        cache_dir=None, 
+                        *inputs,
+                        **kwargs):
+        """
+        Instantiate a PreTrainedBertModel from a pre-trained model file.
+        Download and cache the pre-trained model file if needed.
+
+        Args:
+            tokenizer_model_name (`str`):
+                Name of the model associated with the tokenizer
+            cache_dir (`str`):
+                The directory that contains the vocab files, or will receive the downloaded vocab files
+        """
+        if cache_dir is None:
+            # cache_dir = os.path.join(os.path.dirname(__file__), 'vocabs')
+            cache_dir = "/root/.cache/FlagAI/"+tokenizer_model_name
+        tokenizer_class = ""
+        # search the cache directory for certain files
+
+        if os.path.exists(cache_dir):
+            files = os.listdir(cache_dir)
+            if SP_MODEL_FILE in files:
+                tokenizer_class = "sp"
+            elif MERGES_FILE in files:
+                tokenizer_class = "bpe"
+            elif VOCAB_FILE in files:
+                tokenizer_class = "wp"
+        if tokenizer_class == "":
+            print("downloading model %s from ModelHub"%tokenizer_model_name)
+            files = _get_model_files(tokenizer_model_name)
+            model_id = _get_model_id(tokenizer_model_name)
+            if SP_MODEL_FILE in files:
+                tokenizer_class = "sp"
+                _get_vocab_path(cache_dir + '/', SP_MODEL_FILE, model_id, rank=0)
+            elif MERGES_FILE in files:
+                tokenizer_class = "bpe"
+                _get_vocab_path(cache_dir + '/', MERGES_FILE, model_id, rank=0)
+                if VOCAB_JSON_FILE in files:
+                    _get_vocab_path(cache_dir + '/', VOCAB_JSON_FILE, model_id, rank=0)
+            elif VOCAB_FILE in files:
+                tokenizer_class = "wp"
+                _get_vocab_path(cache_dir + '/', VOCAB_FILE, model_id, rank=0)
+            else:
+                raise FileNotFoundError("Error: no tokenizer files")
+        resolved_vocab_json_file = os.path.join(cache_dir, VOCAB_JSON_FILE) if VOCAB_JSON_FILE in files else None
+        resolved_vocab_file = os.path.join(cache_dir, VOCAB_FILE)
+        resolved_merges_file = os.path.join(cache_dir, MERGES_FILE)
+        resolved_sp_file = os.path.join(cache_dir, SP_MODEL_FILE)
+        if tokenizer_class == "wp":
+            return cls(vocab_file=resolved_vocab_file, tokenizer_class=tokenizer_class,
+                       tokenizer_model_name=tokenizer_model_name, cache_dir=cache_dir, *inputs, **kwargs)
+        elif tokenizer_class == "bpe":
+            return cls(vocab_file=resolved_vocab_json_file, merges_file=resolved_merges_file, tokenizer_class=tokenizer_class,
+                    tokenizer_model_name=tokenizer_model_name, cache_dir=cache_dir, *inputs, **kwargs)
+        elif tokenizer_class == "sp":
+            return cls(sp_model_file=resolved_sp_file, tokenizer_class=tokenizer_class,
+                       tokenizer_model_name=tokenizer_model_name, cache_dir=cache_dir, *inputs, **kwargs)
+        else:
+            raise NotImplementedError("Cannot find a tokenizer class that matches the files settings in the directory or ModelHub")
+
+
+    def __init__(self,
+                 vocab_file=None,
+                 merges_file=None,
+                 sp_model_file=None,
+                 tokenizer_class=None,
+                 tokenizer_model_name=None,
+                 cache_dir=None,
+                 *inputs,
+                 **kwargs):
+
+        self.vocab_file = vocab_file
+        self.merges_file = merges_file
+        self.sp_model_file = sp_model_file
+        self.tokenizer_class = tokenizer_class
+        self.tokenizer_model_name = tokenizer_model_name
+        self.cache_dir = cache_dir
+        self.deprecation_warnings = (
+            {}
+        )
\ No newline at end of file
diff --git a/flagai/data/tokenizer/uni_tokenizer/bpe_tokenizer.py b/flagai/data/tokenizer/uni_tokenizer/bpe_tokenizer.py
new file mode 100644
index 00000000..60952ec4
--- /dev/null
+++ b/flagai/data/tokenizer/uni_tokenizer/bpe_tokenizer.py
@@ -0,0 +1,343 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# coding=utf-8
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for using and training tokenizers (char, wordpiece, sentencepiece)"""
+# from collections import namedtuple
+# import itertools
+
+import logging
+import regex as re
+import json
+from typing import Union, List
+import torch
+import html
+import os
+import ftfy
+from functools import lru_cache
+import regex as re
+logger = logging.getLogger(__name__)
+# from flagai.data.tokenizer.glm_10b_en.glm_10b_en_tokenizer import bytes_to_unicode, get_pairs
+import sys
+try:
+    from functools import lru_cache
+except ImportError:
+    # Just a dummy decorator to get the checks to run on python2
+    # because honestly I don't want to support a byte-level unicode BPE tokenizer on python 2 right now.
+    def lru_cache():
+        return lambda func: func
+
+
+class BPETokenizer(object):
+    def __init__(self,
+                 vocab_file,
+                 merges_file,
+                 errors='replace',
+                 max_len=None,
+                 **kwargs):
+        super().__init__(**kwargs)
+        self.max_len = max_len if max_len is not None else int(1e12)
+        
+        self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        with open(merges_file, encoding='utf-8') as file:
+            bpe_data = file.read().split('\n')[1:-1]
+        bpe_merges = [tuple(merge.split()) for merge in bpe_data]
+        # file.close()
+        if not vocab_file:
+            vocab = list(bytes_to_unicode().values())
+            vocab = vocab + [v for v in vocab]
+            for merge in bpe_merges:
+                vocab.append(''.join(merge))
+            self.encoder = dict(zip(vocab, range(len(vocab))))
+        else:
+            with open(vocab_file) as file:
+                self.encoder = json.load(file)
+        self.decoder = {v: k for k, v in self.encoder.items()}
+
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        self.cache = {}
+        # self.cache = {t:t for t in special_tokens}
+
+        self.pat = re.compile(
+            r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+        )
+
+        self.special_tokens = {}
+        self.special_tokens_decoder = {}
+        # self.set_special_tokens(special_tokens)
+
+    @property
+    def vocab_size(self):
+        return len(self.encoder)
+
+    def get_vocab(self):
+        return dict(self.encoder)
+
+    def __len__(self):
+        return len(self.encoder) + len(self.special_tokens)
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token
+
+        while True:
+            bigram = min(
+                pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word) - 1 and word[
+                        i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def tokenize(self, text):
+        """ Tokenize a string. """
+        bpe_tokens = []
+        for token in re.findall(self.pat, text):
+            if sys.version_info[0] == 2:
+                token = ''.join(self.byte_encoder[ord(b)] for b in token)
+            else:
+                token = ''.join(self.byte_encoder[b]
+                                for b in token.encode('utf-8'))
+            bpe_tokens.extend(bpe_token
+                              for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def convert_token_to_id(self, token):
+        """ Converts a sequence of tokens into ids using the vocab. """
+        return self.encoder.get(token, 0)
+
+    def convert_tokens_to_ids(self, tokens):
+        """ Converts a sequence of tokens into ids using the vocab. """
+        ids = []
+        for token in tokens:
+            ids.append(self.convert_token_to_id(token))
+        if len(ids) > self.max_len:
+            logger.warning(
+                "Token indices sequence length is longer than the specified maximum "
+                " sequence length for this OpenAI GPT model ({} > {}). Running this"
+                " sequence through the model will result in indexing errors".
+                format(len(ids), self.max_len))
+        return ids
+
+    def convert_id_to_token(self, id):
+        """Converts a sequence of ids in BPE tokens using the vocab."""
+        return self.decoder[id]
+
+    def convert_ids_to_tokens(self, ids, skip_special_tokens=False):
+        """Converts a sequence of ids in BPE tokens using the vocab."""
+        tokens = []
+        for i in ids:
+            tokens.append(self.decoder[i])
+        return tokens
+
+    def convert_tokens_to_string(self, tokens, all_command_token={}):
+        """Converts a sequence of tokens (string) in a single string."""
+        text = "".join(tokens)
+        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
+        return text
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    _chr = chr if sys.version_info[0] == 2 else chr
+    bs = list(range(ord("!"),
+                    ord("~") + 1)) + list(range(
+                        ord("¡"),
+                        ord("¬") + 1)) + list(range(ord("®"),
+                                                    ord("ÿ") + 1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [_chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+def whitespace_clean(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+class MMBPETokenizer(BPETokenizer):
+    def __init__(self,
+                vocab_file,
+                merges_file,
+                errors='replace',
+                max_len=None,
+                special_tokens=None,
+                **kwargs):
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        merges = open(merges_file).read().split('\n')
+        merges = merges[1:49152-256-2+1]
+        merges = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v+'</w>' for v in vocab]
+        for merge in merges:
+            vocab.append(''.join(merge))
+        if not special_tokens:
+            special_tokens = ['<start_of_text>', '<end_of_text>']
+        else:
+            special_tokens = ['<start_of_text>', '<end_of_text>'] + special_tokens
+        vocab.extend(special_tokens)
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {t:t for t in special_tokens}
+        special = "|".join(special_tokens)
+        self.pat = re.compile(special + r"""|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
+
+        # self.vocab_size = len(self.encoder)
+        # self.all_special_ids = [self.encoder[t] for t in special_tokens]
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + ( token[-1] + '</w>',)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token+'</w>'
+
+        while True:
+            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word)-1 and word[i+1] == second:
+                    new_word.append(first+second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
+        return text
+
+    def tokenize(self, texts: Union[str, List[str]], sot_token: int, eot_token: int, context_length: int = 77) -> torch.LongTensor:
+        """
+        Returns the tokenized representation of given input string(s)
+
+        Parameters
+        ----------
+        texts : Union[str, List[str]]
+            An input string or a list of input strings to tokenize
+        context_length : int
+            The context length to use; all CLIP models use 77 as the context length
+
+        Returns
+        -------
+        A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
+        """
+        if isinstance(texts, str):
+            texts = [texts]
+
+        all_tokens = [[sot_token] + self.encode(text) + [eot_token] for text in texts]
+        result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+        for i, tokens in enumerate(all_tokens):
+            if len(tokens) > context_length:
+                tokens = tokens[:context_length]  # Truncate
+            result[i, :len(tokens)] = torch.tensor(tokens)
+        return result
\ No newline at end of file
diff --git a/flagai/data/tokenizer/uni_tokenizer/properties.py b/flagai/data/tokenizer/uni_tokenizer/properties.py
new file mode 100644
index 00000000..78499629
--- /dev/null
+++ b/flagai/data/tokenizer/uni_tokenizer/properties.py
@@ -0,0 +1,5 @@
+VOCAB_FILE = 'vocab.txt'
+VOCAB_JSON_FILE = 'vocab.json'
+MERGES_FILE = 'merges.txt'
+SP_MODEL_FILE = 'spiece.model'
+SPECIAL_TOKENS_NAME = 'special_tokens.txt'
\ No newline at end of file
diff --git a/flagai/data/tokenizer/uni_tokenizer/sp_tokenizer.py b/flagai/data/tokenizer/uni_tokenizer/sp_tokenizer.py
new file mode 100644
index 00000000..78c491af
--- /dev/null
+++ b/flagai/data/tokenizer/uni_tokenizer/sp_tokenizer.py
@@ -0,0 +1,67 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# coding=utf-8
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for using and training tokenizers (char, wordpiece, sentencepiece)"""
+
+import logging
+logger = logging.getLogger(__name__)
+import sentencepiece as spm
+
+
+class SentencePieceTokenizer(object):
+    def __init__(self, model_path):
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.Load(model_path)
+
+    @property
+    def vocab_size(self):
+        return self.sp_model.get_piece_size()
+
+    def get_vocab(self):
+        vocab = {self.convert_id_to_token(i): i for i in range(self.vocab_size)}
+        # vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def tokenize(self, text):
+        return self.sp_model.EncodeAsPieces(text)
+
+    def convert_tokens_to_ids(self, tokens):
+        return [self.sp_model.PieceToId(token) for token in tokens]
+
+    def convert_token_to_id(self, token):
+        return self.sp_model.PieceToId(token)
+
+    def convert_id_to_token(self, idx):
+        return self.sp_model.IdToPiece(int(idx))
+
+    def convert_ids_to_tokens(self, idxs):
+        return [self.sp_model.IdToPiece(idx) for idx in idxs]
+
+    def convert_tokens_to_string(self, tokens, all_command_token={}):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ""
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in all_command_token:
+                out_string += self.sp_model.decode_pieces(current_sub_tokens) + token + " "
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+        out_string += self.sp_model.decode_pieces(current_sub_tokens)
+        return out_string.strip()
+
diff --git a/flagai/data/tokenizer/uni_tokenizer/tokenizer.py b/flagai/data/tokenizer/uni_tokenizer/tokenizer.py
new file mode 100644
index 00000000..2573b71d
--- /dev/null
+++ b/flagai/data/tokenizer/uni_tokenizer/tokenizer.py
@@ -0,0 +1,601 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# coding=utf-8
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for using and training tokenizers (char, wordpiece, sentencepiece)"""
+# from collections import namedtuple
+# import itertools
+
+
+import itertools
+import logging
+logger = logging.getLogger(__name__)
+from flagai.data.tokenizer.tokenizer import CommandToken
+from flagai.data.tokenizer.uni_tokenizer.wp_tokenizer import WordpieceTokenizer
+from flagai.data.tokenizer.uni_tokenizer.bpe_tokenizer import BPETokenizer, MMBPETokenizer
+from flagai.data.tokenizer.uni_tokenizer.sp_tokenizer import SentencePieceTokenizer
+from flagai.data.tokenizer.uni_tokenizer.base_tokenizer import BaseTokenizer
+from typing import List, Union, Optional
+import unicodedata
+
+
+def is_control(ch):
+    """控制类字符判断
+    https://en.wikipedia.org/wiki/Control_character
+    https://www.fileformat.info/info/unicode/category/Cc/index.htm
+    https://www.fileformat.info/info/unicode/category/Cf/index.htm
+    
+    """
+    return unicodedata.category(ch) in ('Cc', 'Cf')
+
+
+
+class Tokenizer(BaseTokenizer):
+    def __init__(self,
+                 add_block_symbols=True,
+                 add_sentinel_token=0,
+                 add_task_mask=True,
+                 add_decoder_mask=False,
+                 fix_command_token=True,
+                 **kwargs):
+        super().__init__(**kwargs)
+
+        if self.tokenizer_class == "wp":
+            self.text_tokenizer = WordpieceTokenizer(self.vocab_file)
+        elif self.tokenizer_class == "bpe":
+            if self.tokenizer_model_name.startswith('clip'):
+                self.text_tokenizer = MMBPETokenizer(self.vocab_file, self.merges_file)
+            else:
+                self.text_tokenizer = BPETokenizer(self.vocab_file, self.merges_file)
+        elif self.tokenizer_class == "sp":
+            self.text_tokenizer = SentencePieceTokenizer(self.sp_model_file)
+        else:
+            raise NotImplementedError("cannot assign a tokenize class")
+
+        self.is_glm = self.tokenizer_model_name.startswith('GLM')
+        # self.is_clip = self.tokenizer_model_name.startswith('clip')
+        self.num_tokens = self.text_tokenizer.vocab_size
+
+        if self.tokenizer_class == "wp":
+            # set command tokens from wordpiece tokenizer values
+            self.num_command_tokens = 6
+            self.num_text_tokens = self.num_tokens - 5
+            self.num_type_tokens = 2
+
+
+            try:
+                self._command_tokens = [
+                    CommandToken('pad', '[PAD]', self.text_tokenizer.convert_token_to_id('[PAD]')),
+                    CommandToken('cls', '[CLS]', self.text_tokenizer.convert_token_to_id('[CLS]')),
+                    CommandToken('MASK', '[MASK]',
+                                 self.text_tokenizer.convert_token_to_id('[MASK]')),
+                    CommandToken('unk', '[UNK]', self.text_tokenizer.convert_token_to_id('[UNK]')),
+                    CommandToken('sep', '[SEP]', self.text_tokenizer.convert_token_to_id('[SEP]')),
+                    CommandToken('eos', '[PAD]', self.text_tokenizer.convert_token_to_id('[PAD]')),
+                ]
+            except KeyError:
+                self._command_tokens = [
+                    CommandToken('pad', '[PAD]', self.text_tokenizer.convert_token_to_id('<pad>')),
+                    CommandToken('cls', '[CLS]', self.text_tokenizer.convert_token_to_id('
')),
+                    CommandToken('MASK', '[MASK]',
+                                 self.text_tokenizer.convert_token_to_id('<mask>')),
+                    CommandToken('unk', '[UNK]', self.text_tokenizer.convert_token_to_id('<unk>')),
+                    CommandToken('sep', '[SEP]', self.text_tokenizer.convert_token_to_id('<sep>')),
+                    CommandToken('eos', '[PAD]', self.text_tokenizer.convert_token_to_id('
')),
+                ]
+            if add_block_symbols:
+                self._command_tokens.extend([
+                    CommandToken('sop', '
', self.num_tokens),
+                    CommandToken('eop', '
', self.num_tokens + 1)
+                ])
+                self.num_tokens += 2
+                self.num_command_tokens += 2
+                if add_task_mask:
+                    self._command_tokens.extend([
+                        CommandToken('gMASK', '[gMASK]', self.num_tokens),
+                        CommandToken('sMASK', '[sMASK]', self.num_tokens + 1)
+                    ])
+                    self.num_tokens += 2
+                    self.num_command_tokens += 2
+                if add_decoder_mask:
+                    self._command_tokens.extend(
+                        [CommandToken('dBLOCK', '[dBLOCK]', self.num_tokens)])
+                    self.num_tokens += 1
+                    self.num_command_tokens += 1
+            if add_sentinel_token > 0:
+                for i in range(1, add_sentinel_token):
+                    self._command_tokens.extend([
+                        CommandToken(f'MASK{i}', f'[MASK{i}]', self.num_tokens),
+                        CommandToken(f'sop{i}', f'<|startofpiece{i}|>',
+                                     self.num_tokens + 1)
+                    ])
+                    self.num_tokens += 2
+                    self.num_command_tokens += 2
+        elif self.tokenizer_class == "bpe":
+            if self.tokenizer_model_name.startswith('roberta'):
+                self.num_command_tokens = 6
+                self.num_text_tokens = self.num_tokens - 3
+                self._command_tokens = [
+                    CommandToken('pad', '
',
+                                 self.text_tokenizer.convert_token_to_id('
')),
+                    CommandToken('eos', '
',
+                                 self.text_tokenizer.convert_token_to_id('
')),
+                    CommandToken('sep', '[SEP]',
+                                 self.text_tokenizer.convert_token_to_id('
')),
+                    CommandToken('cls', '[CLS]',
+                                 self.text_tokenizer.convert_token_to_id('
')),
+                    CommandToken('MASK',
+                                 '[MASK]',
+                                 self.text_tokenizer.convert_token_to_id('<mask>'),
+                                 lstrip=True),
+                    CommandToken('unk', '[UNK]',
+                                 self.text_tokenizer.convert_token_to_id('<unk>'))
+                ]
+                if add_block_symbols:
+                    self._command_tokens.extend([
+                        CommandToken('sop', '
', self.num_tokens),
+                        CommandToken('eop', '
', self.num_tokens + 1)
+                    ])
+                    self.num_tokens += 2
+                    self.num_command_tokens += 2
+            elif self.tokenizer_model_name.startswith('clip'):
+                self.num_command_tokens = 2
+                self._command_tokens = [
+                    CommandToken('sot', '<start_of_text>',
+                                 self.text_tokenizer.convert_token_to_id('
')),
+                    CommandToken('eot', '<end_of_text>',
+                                 self.text_tokenizer.convert_token_to_id('
')),
+                ]
+                self.num_tokens += self.num_command_tokens 
+            else:
+                self.num_command_tokens = 2
+                self.num_text_tokens = self.num_tokens - 1
+                self._command_tokens = [
+                    CommandToken('pad', '
',
+                                 self.text_tokenizer.convert_token_to_id('
')),
+                    CommandToken('eos', '
',
+                                 self.text_tokenizer.convert_token_to_id('
'))
+                ]
+                if add_block_symbols:
+                    if self.tokenizer_model_name.startswith('GLM'):
+                        unk_token_id = self.num_tokens + 5
+                        cls_token_id = self.num_tokens + 2
+                        num_tokens_to_add = 5
+                    else:
+                        unk_token_id = self.text_tokenizer.convert_token_to_id('
')
+                        cls_token_id = self.text_tokenizer.convert_token_to_id('
')
+                        num_tokens_to_add = 4
+                    self._command_tokens.extend([
+                        CommandToken('sop', '
', self.num_tokens),
+                        CommandToken('eop', '
', self.num_tokens + 1),
+                        CommandToken('cls', '[CLS]', cls_token_id),
+                        CommandToken('MASK',
+                                     '[MASK]',
+                                     self.num_tokens + 3,
+                                     lstrip=True),
+                        CommandToken('sep', '[SEP]', self.num_tokens + 4),
+                        CommandToken('unk', '[UNK]', unk_token_id)
+                    ])
+                    self.num_tokens += num_tokens_to_add
+                    self.num_command_tokens += 6
+            if add_block_symbols:
+                if add_task_mask:
+                    self._command_tokens.extend([
+                        CommandToken('gMASK',
+                                     '[gMASK]',
+                                     self.num_tokens,
+                                     lstrip=True),
+                        CommandToken('sMASK',
+                                     '[sMASK]',
+                                     self.num_tokens + 1,
+                                     lstrip=True)
+                    ])
+                    self.num_tokens += 2
+                    self.num_command_tokens += 2
+                if add_decoder_mask:
+                    self._command_tokens.extend(
+                        [CommandToken('dBLOCK', '[dBLOCK]', self.num_tokens)])
+                    self.num_tokens += 1
+                    self.num_command_tokens += 1
+        elif self.tokenizer_class == "sp":
+            self.num_command_tokens = 0
+            self.num_text_tokens = self.text_tokenizer.vocab_size
+            self.num_tokens = self.num_text_tokens
+
+            if self.tokenizer_model_name.startswith('GLM'):
+                pad_token_id = self.num_tokens
+                eos_token_id = self.num_tokens
+                unk_token_id = self.num_tokens + 4
+                num_tokens_to_add = 4
+            else:
+                pad_token_id = self.text_tokenizer.convert_token_to_id('<pad>')
+                eos_token_id = self.text_tokenizer.convert_token_to_id('
')
+                unk_token_id = self.text_tokenizer.convert_token_to_id('<unk>')
+                num_tokens_to_add = 3
+            self._command_tokens = [
+                CommandToken('pad', '
', pad_token_id),
+                CommandToken('eos', '
', eos_token_id),
+                CommandToken('sep', '[SEP]', self.num_text_tokens + 1),
+                CommandToken('cls', '[CLS]', self.num_text_tokens + 2),
+                CommandToken('MASK',
+                             '[MASK]',
+                             self.num_text_tokens + 3,
+                             lstrip=True),
+                CommandToken('unk', '[UNK]', unk_token_id)
+            ]
+            self.num_tokens += num_tokens_to_add
+            self.num_command_tokens += 6
+            if add_block_symbols:
+                self._command_tokens.extend([
+                    CommandToken('sop', '
', self.num_tokens + 1),
+                    CommandToken('eop', '
', self.num_tokens + 2)
+                ])
+                if fix_command_token:
+                    self.num_tokens += 3
+                else:
+                    self.num_tokens += 2
+                self.num_command_tokens += 2
+                if add_task_mask:
+                    if fix_command_token:
+                        self._command_tokens.extend([
+                            CommandToken('sMASK',
+                                         '[sMASK]',
+                                         self.num_tokens,
+                                         lstrip=True),
+                            CommandToken('gMASK',
+                                         '[gMASK]',
+                                         self.num_tokens + 1,
+                                         lstrip=True)
+                        ])
+                    else:
+                        self._command_tokens.extend([
+                            CommandToken('gMASK',
+                                         '[gMASK]',
+                                         self.num_tokens,
+                                         lstrip=True),
+                            CommandToken('sMASK',
+                                         '[sMASK]',
+                                         self.num_tokens + 1,
+                                         lstrip=True)
+                        ])
+                    self.num_tokens += 2
+                    self.num_command_tokens += 2
+                if add_decoder_mask:
+                    self._command_tokens.extend(
+                        [CommandToken('dBLOCK', '[dBLOCK]', self.num_tokens)])
+                    self.num_tokens += 1
+                    self.num_command_tokens += 1
+        self.command_name_map = {tok.name: tok for tok in self._command_tokens}
+        self.command_token_map = {
+            tok.token: tok
+            for tok in self._command_tokens
+        }
+        self.command_id_map = {tok.Id: tok for tok in self._command_tokens}
+        self._command_token_tokens = list(self.command_token_map.keys())
+
+    def get_vocab(self):
+        return self.text_tokenizer.get_vocab()
+
+    def get_command_id(self, name):
+        """get command token corresponding to `name`"""
+        return self.command_name_map[name].Id
+    
+    def rematch(self, text, tokens):
+        text = text.lower()
+
+        normalized_text, char_mapping = '', []
+        for i, ch in enumerate(text):
+            if True:
+                ch = unicodedata.normalize('NFD', ch)
+                ch = ''.join([c for c in ch if unicodedata.category(c) != 'Mn'])
+            ch = ''.join([
+                c for c in ch
+                if not (ord(c) == 0 or ord(c) == 0xfffd or is_control(c))
+            ])
+            normalized_text += ch
+            char_mapping.extend([i] * len(ch))
+
+        text, token_mapping, offset = normalized_text, [], 0
+        for token in tokens:
+            start = text[offset:].index(token) + offset
+            end = start + len(token)
+            token_mapping.append(char_mapping[start:end])
+            offset = end
+        return token_mapping
+
+    def _encode(self, text):
+        tokens = self.text_tokenizer.tokenize(text)
+        ids = self.text_tokenizer.convert_tokens_to_ids(tokens)
+        return ids
+
+    def convert_tokens_to_ids(self, tokens):
+        return self.text_tokenizer.convert_tokens_to_ids(tokens)
+
+    def convert_ids_to_tokens(self, ids):
+        return self.text_tokenizer.convert_ids_to_tokens(ids)
+
+    def EncodeAsTokens(self, text, process_fn=None):
+        """convert wordpiece token to Id"""
+        processed_text = text
+        if process_fn is not None:
+            processed_text = process_fn(processed_text)
+        tokens = self.text_tokenizer.tokenize(processed_text)
+        return tokens
+
+    def IdToToken(self, id):
+        """convert Id to sentencpiece token"""
+        if isinstance(id, (CommandToken)):
+            return id.token
+        if id in self.command_id_map:
+            return self.command_id_map[id].token
+        return self.text_tokenizer.convert_id_to_token(id)
+
+    def TokenToId(self, token):
+        """convert sentencpiece token to Id"""
+        token = token.lower()
+        if isinstance(token, (CommandToken)):
+            return token.Id
+        try:
+            return self.text_tokenizer.convert_token_to_id(token)
+        except KeyError:
+            return self.text_tokenizer.convert_token_to_id(token.strip())
+
+    def DecodeIds(self, ids):
+        """converts ids to wordpiece tokens and joins them as a text string"""
+        tokens = []
+        for id in ids:
+            if id in self.command_id_map:
+                tokens.append(self.command_id_map[id].token)
+            else:
+                try:
+                    tokens.extend(self.text_tokenizer.convert_ids_to_tokens([id]))
+                except KeyError:
+                    pass
+        return self.text_tokenizer.convert_tokens_to_string(tokens, self.command_token_map)
+
+    def encode(self, text):
+        return self.text_tokenizer.convert_tokens_to_ids(self.text_tokenizer.tokenize(text))
+
+    def decode(self, ids):
+        return self.DecodeIds(ids)
+
+    def DecodeTokens(self, tokens):
+        """converts wordpiece tokens to a text string"""
+        return self.text_tokenizer.convert_tokens_to_string(tokens, self.command_token_map)
+
+    def EncodeAsIds(self, text, process_fn=None):
+        """
+        encode text using text tokenizer and shift Id values for command tokens
+        """
+        processed_text = text
+        if process_fn is not None:
+            processed_text = process_fn(processed_text)
+
+        def split_on_token(tok_extended: CommandToken, text):
+            result = []
+            tok = tok_extended.token
+            split_text = text.split(tok)
+            for i, sub_text in enumerate(split_text):
+                # CommandToken can control whitespace stripping around them.
+                # We use them for GPT2 and Roberta to have different behavior depending on the special token
+                # Cf. https://github.com/huggingface/transformers/pull/2778
+                # and https://github.com/huggingface/transformers/issues/3788
+                # Strip white spaces on the right
+                if tok_extended.rstrip and i > 0:
+                    # A bit counter-intuitive but we strip the left of the string
+                    # since tok_extended.rstrip means the special token is eating all white spaces on its right
+                    sub_text = sub_text.lstrip()
+                # Strip white spaces on the left
+                if tok_extended.lstrip and i < len(split_text) - 1:
+                    sub_text = sub_text.rstrip()  # Opposite here
+
+                if i == 0 and not sub_text:
+                    result.append(tok)
+                elif i == len(split_text) - 1:
+                    if sub_text:
+                        result.append(sub_text)
+                    else:
+                        pass
+                else:
+                    if sub_text:
+                        result.append(sub_text)
+                    result.append(tok)
+            return result
+
+        def split_on_tokens(tok_list, text):
+            if not text.strip():
+                return []
+            if not tok_list:
+                return self.encode(text)
+
+            tokenized_text = []
+            text_list = [text]
+            for tok in tok_list:
+                tokenized_text = []
+                for sub_text in text_list:
+                    if sub_text not in self._command_token_tokens:
+                        tokenized_text.extend(split_on_token(tok, sub_text))
+                    else:
+                        tokenized_text.append(sub_text)
+                text_list = tokenized_text
+
+            return list(
+                itertools.chain.from_iterable(
+                    (self._encode(token)
+                     if token not in self._command_token_tokens else
+                     [self.command_token_map[token].Id]
+                     for token in tokenized_text)))
+
+        no_split_tokens = self._command_tokens
+        Ids = split_on_tokens(no_split_tokens, processed_text)
+        return Ids
+
+    def CommandTokenIds(self, exception=None):
+        result = []
+        for s in self._command_tokens:
+            if not exception or (exception and s.name not in exception):
+                result.append(s.Id)
+        return (result)
+
+
+    def encode_plus_non_glm(
+        self,
+        text,
+        second_text=None,
+        truncation=True,
+        max_length=None,
+    ):
+
+        def get_input_ids(text):
+            tokens = self.text_tokenizer.tokenize(text)
+            return self.text_tokenizer.convert_tokens_to_ids(tokens)
+
+        first_ids = get_input_ids(text)
+        second_ids = get_input_ids(
+            second_text) if second_text is not None else None
+
+        return self.prepare_for_model(
+            first_ids,
+            pair_ids=second_ids,
+            truncation=truncation,
+            max_length=max_length,
+        )
+
+
+    def prepare_for_model(
+        self,
+        ids: List[int],
+        pair_ids: Optional[List[int]] = None,
+        truncation: Union[bool, str] = True,
+        max_length: Optional[int] = None,
+    ):
+
+        pair = bool(pair_ids is not None)
+        len_ids = len(ids)
+        len_pair_ids = len(pair_ids) if pair else 0
+
+        encoded_inputs = {}
+        total_len = len_ids + len_pair_ids + 3
+
+        # Truncation: Handle max sequence length
+        if truncation is True and (max_length is not None
+                                   and total_len > max_length):
+            self.truncate_sequence(
+                max_length,
+                ids,
+                pair_ids,
+                pop_index=-1,
+            )
+
+
+        sequence = ids + pair_ids if pair else ids
+        token_type_ids = [0] * len(ids) + ([0] *
+                                           len(pair_ids) if pair else [])
+
+        encoded_inputs["input_ids"] = sequence
+        encoded_inputs["token_type_ids"] = token_type_ids
+        return encoded_inputs
+
+    def encode_plus(  #for Seq2seq
+            self,
+            source_text: str,
+            target_text=None,
+            second_text=None,
+            truncation=True,
+            max_length=None,
+    ):
+        if not self.tokenizer_model_name.startswith("GLM"):
+            return self.encode_plus_non_glm(source_text, second_text, truncation, max_length)
+        sop_id = self.get_command_id('sop')  #start of piece
+        eop_id = self.get_command_id('eop')  #end of piece
+        sep_id = self.get_command_id('sep')  #seperation
+
+        source_tokens = self.EncodeAsIds(source_text)
+        source_tokens = [sop_id] + source_tokens + [sep_id]
+
+        # no pading for consistency
+        len_source = len(source_tokens)
+        sop_pos = source_tokens.index(sop_id)
+        loss_mask = [0] * len_source
+        block_position_ids = [0] * len_source
+        position_ids = list(range(len_source))
+
+        if target_text:
+            target_tokens = self.EncodeAsIds(target_text)
+            target_tokens = target_tokens + [eop_id]
+            loss_mask += [1] * len(target_tokens)
+            block_position_ids += [0] * len(target_tokens)
+            position_ids += [x + len_source for x in range(len(target_tokens))]
+            tokens = source_tokens + target_tokens
+            position_ids = [position_ids[:-1], block_position_ids[:-1]]
+            sample = {
+                'input_ids': tokens[:-1],
+                'target_ids': tokens[1:],
+                'attention_mask': sop_pos,
+                'loss_mask': loss_mask[:-1],
+                "position_ids": position_ids
+            }
+        else:
+            position_ids = [position_ids, block_position_ids]
+            sample = {
+                'input_ids': source_tokens,
+                'attention_mask': sop_pos,
+                "position_ids": position_ids,
+                'loss_mask': loss_mask,
+            }
+        return sample
+
+    @staticmethod
+    def truncate_sequence(max_length,
+                          first_sequence,
+                          second_sequence=None,
+                          pop_index=-1):
+
+        if second_sequence is None:
+            second_sequence = []
+
+        while True:
+            total_length = len(first_sequence) + len(second_sequence)
+            if total_length <= max_length:
+                break
+            elif len(first_sequence) > len(second_sequence):
+                first_sequence.pop(pop_index)
+            else:
+                second_sequence.pop(pop_index)
+
+    def tokenize_as_tensor(self, texts):
+        """
+        Returns the tokenized representation of given input string(s)
+
+        Parameters
+        ----------
+        texts : Union[str, List[str]]
+            An input string or a list of input strings to tokenize
+        context_length : int
+            The context length to use; all CLIP models use 77 as the context length
+
+        Returns
+        -------
+        A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
+        """
+        sot_token = self.get_command_id('sot')
+        eot_token = self.get_command_id('eot')
+        return self.text_tokenizer.tokenize(texts, sot_token=sot_token, eot_token=eot_token)
+
+    def tokenize(self, texts):
+        return self.text_tokenizer.tokenize(texts)
+
+
diff --git a/flagai/data/tokenizer/uni_tokenizer/wp_tokenizer.py b/flagai/data/tokenizer/uni_tokenizer/wp_tokenizer.py
new file mode 100644
index 00000000..0fc23114
--- /dev/null
+++ b/flagai/data/tokenizer/uni_tokenizer/wp_tokenizer.py
@@ -0,0 +1,337 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# coding=utf-8
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for using and training tokenizers (char, wordpiece, sentencepiece)"""
+# from collections import namedtuple
+# import itertools
+
+
+
+import logging
+logger = logging.getLogger(__name__)
+import os
+# from flagai.data.tokenizer.glm_large_en.wordpiece import load_vocab, BasicTokenizer, whitespace_tokenize
+import collections
+import unicodedata
+import json
+
+
+class WordpieceTokenizer(object):
+    def __init__(self, vocab_file=None, do_basic_tokenize=True,
+                         do_lower_case=True, max_len=None,
+                         never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]"),
+                         unk_token="[UNK]", max_input_chars_per_word=100, *input, **kwargs):
+        if not os.path.isfile(vocab_file):
+            raise ValueError(
+                "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained "
+                "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
+                    .format(vocab_file))
+        self.vocab = load_vocab(vocab_file)
+        self.ids_to_tokens = collections.OrderedDict([
+            (ids, tok) for tok, ids in self.vocab.items()
+        ])
+        self.do_basic_tokenize = do_basic_tokenize
+        if do_basic_tokenize:
+            self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case,
+                                                  never_split=never_split)
+        self.max_len = max_len if max_len is not None else int(1e12)
+        self.unk_token = unk_token
+        self.max_input_chars_per_word = max_input_chars_per_word
+
+    @property
+    def vocab_size(self):
+        return len(self.vocab)
+
+    def get_vocab(self):
+        return self.vocab
+
+    def word_piece(self, text):
+        """Tokenizes a piece of text into its word pieces.
+        This uses a greedy longest-match-first algorithm to perform tokenization
+        using the given vocabulary.
+        For example:
+          input = "unaffable"
+          output = ["un", "##aff", "##able"]
+        Args:
+          text: A single token or whitespace separated tokens. This should have
+            already been passed through `BasicTokenizer`.
+        Returns:
+          A list of wordpiece tokens.
+        """
+
+        output_tokens = []
+        for token in whitespace_tokenize(text):
+            chars = list(token)
+            if len(chars) > self.max_input_chars_per_word:
+                output_tokens.append(self.unk_token)
+                continue
+
+            is_bad = False
+            start = 0
+            sub_tokens = []
+            while start < len(chars):
+                end = len(chars)
+                cur_substr = None
+                while start < end:
+                    substr = "".join(chars[start:end])
+                    if start > 0:
+                        substr = "##" + substr
+                    if substr in self.vocab:
+                        cur_substr = substr
+                        break
+                    end -= 1
+                if cur_substr is None:
+                    is_bad = True
+                    break
+                sub_tokens.append(cur_substr)
+                start = end
+
+            if is_bad:
+                output_tokens.append(self.unk_token)
+            else:
+                output_tokens.extend(sub_tokens)
+        return output_tokens
+
+    def tokenize(self, text):
+        if self.do_basic_tokenize:
+            split_tokens = []
+            for token in self.basic_tokenizer.tokenize(text):
+                for sub_token in self.word_piece(token):
+                    split_tokens.append(sub_token)
+        else:
+            split_tokens = self.word_piece(text)
+        return split_tokens
+
+    def convert_token_to_id(self, token):
+        """ Converts a sequence of tokens into ids using the vocab. """
+        return self.vocab[token]
+
+    def convert_tokens_to_ids(self, tokens):
+        """Converts a sequence of tokens into ids using the vocab."""
+        ids = [self.convert_token_to_id(token) for token in tokens]
+        if len(ids) > self.max_len:
+            logger.warning(
+                "Token indices sequence length is longer than the specified maximum "
+                " sequence length for this BERT model ({} > {}). Running this"
+                " sequence through BERT will result in indexing errors".format(
+                    len(ids), self.max_len))
+        return ids
+
+    def convert_id_to_token(self, id):
+        """Converts a sequence of ids in wordpiece tokens using the vocab."""
+        return self.ids_to_tokens[id]
+
+    def convert_ids_to_tokens(self, ids):
+        """Converts a sequence of ids in wordpiece tokens using the vocab."""
+        return [self.convert_id_to_token(id) for id in ids]
+
+    def convert_tokens_to_string(self, tokens, all_command_token={}):
+        """Converts a sequence of tokens (string) in a single string."""
+        out_string = " ".join(tokens).replace(" ##", "").strip()
+        return out_string
+
+def load_vocab(vocab_file):
+    """Loads a vocabulary file into a dictionary."""
+    vocab = collections.OrderedDict()
+    index = 0
+    with open(vocab_file, "r", encoding="utf-8") as reader:
+        while True:
+            token = reader.readline()
+            # if token.startswith('{') and token.endswith('{'):
+            #     return json.loads(token)
+            if not token:
+                break
+            token = token.strip()
+            vocab[token] = index
+            index += 1
+    return vocab
+
+
+def whitespace_tokenize(text):
+    """Runs basic whitespace cleaning and splitting on a piece of text."""
+    text = text.strip()
+    if not text:
+        return []
+    tokens = text.split()
+    return tokens
+
+
+class BasicTokenizer(object):
+    """Runs basic tokenization (punctuation splitting, lower casing, etc.)."""
+
+
+    def __init__(self,
+                 do_lower_case=True,
+                 never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")):
+        """Constructs a BasicTokenizer.
+        Args:
+          do_lower_case: Whether to lower case the input.
+        """
+        self.do_lower_case = do_lower_case
+        self.never_split = never_split
+
+
+    def tokenize(self, text):
+        """Tokenizes a piece of text."""
+        text = self._clean_text(text)
+        # This was added on November 1st, 2018 for the multilingual and Chinese
+        # models. This is also applied to the English models now, but it doesn't
+        # matter since the English models were not trained on any Chinese data
+        # and generally don't have any Chinese data in them (there are Chinese
+        # characters in the vocabulary because Wikipedia does have some Chinese
+        # words in the English Wikipedia.).
+        text = self._tokenize_chinese_chars(text)
+        orig_tokens = whitespace_tokenize(text)
+        split_tokens = []
+        for token in orig_tokens:
+            if self.do_lower_case and token not in self.never_split:
+                token = token.lower()
+                token = self._run_strip_accents(token)
+            split_tokens.extend(self._run_split_on_punc(token))
+
+        output_tokens = whitespace_tokenize(" ".join(split_tokens))
+        return output_tokens
+
+
+    def _run_strip_accents(self, text):
+        """Strips accents from a piece of text."""
+        text = unicodedata.normalize("NFD", text)
+        output = []
+        for char in text:
+            cat = unicodedata.category(char)
+            if cat == "Mn":
+                continue
+            output.append(char)
+        return "".join(output)
+
+
+    def _run_split_on_punc(self, text):
+        """Splits punctuation on a piece of text."""
+        if text in self.never_split:
+            return [text]
+        chars = list(text)
+        i = 0
+        start_new_word = True
+        output = []
+        while i < len(chars):
+            char = chars[i]
+            if _is_punctuation(char):
+                output.append([char])
+                start_new_word = True
+            else:
+                if start_new_word:
+                    output.append([])
+                start_new_word = False
+                output[-1].append(char)
+            i += 1
+
+        return ["".join(x) for x in output]
+
+
+    def _tokenize_chinese_chars(self, text):
+        """Adds whitespace around any CJK character."""
+        output = []
+        for char in text:
+            cp = ord(char)
+            if self._is_chinese_char(cp):
+                output.append(" ")
+                output.append(char)
+                output.append(" ")
+            else:
+                output.append(char)
+        return "".join(output)
+
+
+    def _is_chinese_char(self, cp):
+        """Checks whether CP is the codepoint of a CJK character."""
+        # This defines a "chinese character" as anything in the CJK Unicode block:
+        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+        #
+        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+        # despite its name. The modern Korean Hangul alphabet is a different block,
+        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+        # space-separated words, so they are not treated specially and handled
+        # like the all of the other languages.
+        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #
+                (cp >= 0x3400 and cp <= 0x4DBF) or  #
+                (cp >= 0x20000 and cp <= 0x2A6DF) or  #
+                (cp >= 0x2A700 and cp <= 0x2B73F) or  #
+                (cp >= 0x2B740 and cp <= 0x2B81F) or  #
+                (cp >= 0x2B820 and cp <= 0x2CEAF) or
+                (cp >= 0xF900 and cp <= 0xFAFF) or  #
+                (cp >= 0x2F800 and cp <= 0x2FA1F)):  #
+            return True
+
+        return False
+
+
+    def _clean_text(self, text):
+        """Performs invalid character removal and whitespace cleanup on text."""
+        output = []
+        for char in text:
+            cp = ord(char)
+            if cp == 0 or cp == 0xfffd or _is_control(char):
+                continue
+            if _is_whitespace(char):
+                output.append(" ")
+            else:
+                output.append(char)
+        return "".join(output)
+
+
+def _is_whitespace(char):
+    """Checks whether `chars` is a whitespace character."""
+    # \t, \n, and \r are technically contorl characters but we treat them
+    # as whitespace since they are generally considered as such.
+    if char == " " or char == "\t" or char == "\n" or char == "\r":
+        return True
+    cat = unicodedata.category(char)
+    if cat == "Zs":
+        return True
+    return False
+
+
+def _is_control(char):
+    """Checks whether `chars` is a control character."""
+    # These are technically control characters but we count them as whitespace
+    # characters.
+    if char == "\t" or char == "\n" or char == "\r":
+        return False
+    cat = unicodedata.category(char)
+    if cat.startswith("C"):
+        return True
+    return False
+
+
+def _is_punctuation(char):
+    """Checks whether `chars` is a punctuation character."""
+    cp = ord(char)
+    # We treat all non-letter/number ASCII as punctuation.
+    # Characters such as "^", "$", and "`" are not in the Unicode
+    # Punctuation class but we treat them as punctuation anyways, for
+    # consistency.
+    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64)
+            or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):
+        return True
+    cat = unicodedata.category(char)
+    if cat.startswith("P"):
+        return True
+    return False
+
+
+
+
diff --git a/flagai/data/tokenizer/wp_tokenizer.py b/flagai/data/tokenizer/wp_tokenizer.py
new file mode 100644
index 00000000..6f163330
--- /dev/null
+++ b/flagai/data/tokenizer/wp_tokenizer.py
@@ -0,0 +1,389 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# coding=utf-8
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for using and training tokenizers (char, wordpiece, sentencepiece)"""
+from collections import namedtuple
+import itertools
+
+
+
+import logging
+logger = logging.getLogger(__name__)
+import os
+from flagai.model.file_utils import _get_model_id, _get_vocab_path
+from flagai.data.tokenizer.glm_large_ch.glm_large_ch import get_encoder
+from flagai.data.tokenizer.glm_10b_en.glm_10b_en_tokenizer import bytes_to_unicode
+from flagai.data.tokenizer.glm_large_en.wordpiece import load_vocab, BasicTokenizer, WordpieceTokenizer
+import collections
+import json
+import re
+
+
+import logging
+logger = logging.getLogger(__name__)
+import os
+from flagai.model.file_utils import _get_model_id, _get_vocab_path
+from flagai.data.tokenizer.glm_large_ch.glm_large_ch import get_encoder
+from flagai.data.tokenizer.glm_10b_en.glm_10b_en_tokenizer import bytes_to_unicode
+from flagai.data.tokenizer.glm_large_en.wordpiece import load_vocab, BasicTokenizer, WordpieceTokenizer
+import collections
+import json
+import re
+
+
+class BaseTokenizer(object):
+    @classmethod
+    def from_pretrained(cls,
+                        pretrained_model_name_or_path,
+                        cache_dir=None,
+                        *inputs,
+                        **kwargs):
+        """
+        Instantiate a PreTrainedBertModel from a pre-trained model file.
+        Download and cache the pre-trained model file if needed.
+        """
+        vocab_file = 'vocab.txt'
+        merges_file = 'merges.txt'
+        sp_file = 'spm.model'
+        if cache_dir is None:
+            cache_dir = os.path.join(os.path.dirname(__file__), 'vocabs')
+        tokenizer_class = "wp"
+        # search the cache directory for certain files
+        if os.path.exists(cache_dir):
+            if os.path.exists(cache_dir + '/' + vocab_file):  # Temporary if statement
+                if os.path.exists(cache_dir + '/' + merges_file):  # Temporary if statement
+                    tokenizer_class = "bpe"
+                else:
+                    tokenizer_class = "wp"
+            elif os.path.exists(cache_dir + '/' + sp_file):
+                tokenizer_class = "sp"
+        else:
+            model_id = _get_model_id(pretrained_model_name_or_path)
+            try:
+                _get_vocab_path(cache_dir + '/', vocab_file, model_id, rank=0)
+                try:
+                    _get_vocab_path(cache_dir + '/', merges_file, model_id, rank=0)
+                    tokenizer_class = "bpe"
+                except:
+                    tokenizer_class = 'wp'
+            except:
+                try:
+                    _get_vocab_path(cache_dir + '/', sp_file, model_id, rank=0)
+                    tokenizer_class = "sp"
+                except:
+                    raise("Error")
+        resolved_vocab_file = os.path.join(cache_dir, vocab_file)
+        resolved_merges_file = os.path.join(cache_dir, merges_file)
+        resolved_sp_file = os.path.join(cache_dir, sp_file)
+        if tokenizer_class == "wp":
+            return cls._from_pretrained(resolved_vocab_file, tokenizer_class, *inputs, **kwargs)
+        elif tokenizer_class == "bpe":
+            return cls._from_pretrained(resolved_vocab_file, resolved_merges_file, tokenizer_class, *inputs, **kwargs)
+        elif tokenizer_class == "sp":
+            return get_encoder(resolved_sp_file, "")
+
+    def __init__(self):
+        self.test = 1
+
+    def _from_pretrained(self, vocab_file=None, do_basic_tokenize=True,
+                            do_lower_case=True, max_len=None,
+                            never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")):
+        raise NotImplementedError(
+            'TextTokenizer tokens property not implemented')
+
+class WordpieceTokenizer(BaseTokenizer):
+    def _from_pretrained(self, vocab_file=None, do_basic_tokenize=True,
+                         do_lower_case=True, max_len=None,
+                         never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")):
+        if not os.path.isfile(vocab_file):
+            raise ValueError(
+                "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained "
+                "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
+                    .format(vocab_file))
+        self.vocab = load_vocab(vocab_file)
+        self.ids_to_tokens = collections.OrderedDict([
+            (ids, tok) for tok, ids in self.vocab.items()
+        ])
+        self.do_basic_tokenize = do_basic_tokenize
+        if do_basic_tokenize:
+            self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case,
+                                                  never_split=never_split)
+        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+        self.max_len = max_len if max_len is not None else int(1e12)
+        self.tokenizer_class = "wp"
+
+    def __init__(self, name, age):
+        self.name = name
+        self.age = age
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+        # if not os.path.isfile(vocab_file):
+        #     raise ValueError(
+        #         "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained "
+        #         "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
+        #         .format(vocab_file))
+        # self.vocab = load_vocab(vocab_file)
+        # self.ids_to_tokens = collections.OrderedDict([
+        #     (ids, tok) for tok, ids in self.vocab.items()
+        # ])
+        # self.do_basic_tokenize = do_basic_tokenize
+        # if do_basic_tokenize:
+        #     self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case,
+        #                                           never_split=never_split)
+        # self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+        # self.max_len = max_len if max_len is not None else int(1e12)
+        # self.tokenizer_class = "wp"
+    #
+    # def set_special_tokens(self, special_tokens):
+    #     """ Add a list of additional tokens to the encoder.
+    #         The additional tokens are indexed starting from the last index of the
+    #         current vocabulary in the order of the `special_tokens` list.
+    #     """
+    #     if not special_tokens:
+    #         self.special_tokens = {}
+    #         self.special_tokens_decoder = {}
+    #         return
+    #     self.special_tokens = dict((tok, len(self.encoder) + i)
+    #                                for i, tok in enumerate(special_tokens))
+    #     self.special_tokens_decoder = {
+    #         v: k
+    #         for k, v in self.special_tokens.items()
+    #     }
+    #     logger.info("Special tokens {}".format(self.special_tokens))
+    #
+    # def _from_pretrained_bpe(self,
+    #              vocab_file,
+    #              merges_file,
+    #              errors='replace',
+    #              special_tokens=None,
+    #              max_len=None):
+    #     self.max_len = max_len if max_len is not None else int(1e12)
+    #     self.encoder = json.load(open(vocab_file))
+    #     self.decoder = {v: k for k, v in self.encoder.items()}
+    #     self.errors = errors  # how to handle errors in decoding
+    #     self.byte_encoder = bytes_to_unicode()
+    #     self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+    #     bpe_data = open(merges_file, encoding='utf-8').read().split('\n')[1:-1]
+    #     bpe_merges = [tuple(merge.split()) for merge in bpe_data]
+    #     self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+    #     self.cache = {}
+    #
+    #     # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
+    #     self.pat = re.compile(
+    #         r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+    #     )
+    #
+    #     self.special_tokens = {}
+    #     self.special_tokens_decoder = {}
+    #     self.set_special_tokens(special_tokens)
+    #     self.tokenizer_class = "bpe"
+    #
+    #
+    # def tokenize(self, text):
+    #     if self.do_basic_tokenize:
+    #         split_tokens = []
+    #         for token in self.basic_tokenizer.tokenize(text):
+    #             for sub_token in self.wordpiece_tokenizer.tokenize(token):
+    #                 split_tokens.append(sub_token)
+    #     else:
+    #         split_tokens = self.wordpiece_tokenizer.tokenize(text)
+    #     return split_tokens
+    #
+    # def convert_tokens_to_ids(self, tokens):
+    #     """Converts a sequence of tokens into ids using the vocab."""
+    #     ids = []
+    #     for token in tokens:
+    #         ids.append(self.vocab[token])
+    #     if len(ids) > self.max_len:
+    #         logger.warning(
+    #             "Token indices sequence length is longer than the specified maximum "
+    #             " sequence length for this BERT model ({} > {}). Running this"
+    #             " sequence through BERT will result in indexing errors".format(
+    #                 len(ids), self.max_len))
+    #     return ids
+    #
+    # def convert_ids_to_tokens(self, ids):
+    #     """Converts a sequence of ids in wordpiece tokens using the vocab."""
+    #     tokens = []
+    #     for i in ids:
+    #         tokens.append(self.ids_to_tokens[i])
+    #     return tokens
+
+
+# from flagai.data.tokenizer.tokenizer import BasicTokenizer
+
+
+# class BaseTokenizer(object):
+#     @classmethod
+#     def from_pretrained(cls,
+#                         pretrained_model_name_or_path,
+#                         cache_dir=None,
+#                         *inputs,
+#                         **kwargs):
+#         """
+#         Instantiate a PreTrainedBertModel from a pre-trained model file.
+#         Download and cache the pre-trained model file if needed.
+#         """
+#         vocab_file = 'vocab.txt'
+#         merges_file = 'merges.txt'
+#         sp_file = 'spm.model'
+#         if cache_dir is None:
+#             cache_dir = os.path.join(os.path.dirname(__file__), 'vocabs')
+#         tokenizer_class = "wp"
+#         # search the cache directory for certain files
+#         if os.path.exists(cache_dir):
+#             if os.path.exists(cache_dir + '/' + vocab_file):  # Temporary if statement
+#                 if os.path.exists(cache_dir + '/' + merges_file):  # Temporary if statement
+#                     tokenizer_class = "bpe"
+#                 else:
+#                     tokenizer_class = "wp"
+#             elif os.path.exists(cache_dir + '/' + sp_file):
+#                 tokenizer_class = "sp"
+#         else:
+#             model_id = _get_model_id(pretrained_model_name_or_path)
+#             try:
+#                 _get_vocab_path(cache_dir + '/', vocab_file, model_id, rank=0)
+#                 try:
+#                     _get_vocab_path(cache_dir + '/', merges_file, model_id, rank=0)
+#                     tokenizer_class = "bpe"
+#                 except:
+#                     tokenizer_class = 'wp'
+#             except:
+#                 try:
+#                     _get_vocab_path(cache_dir + '/', sp_file, model_id, rank=0)
+#                     tokenizer_class = "sp"
+#                 except:
+#                     raise("Error")
+#         resolved_vocab_file = os.path.join(cache_dir, vocab_file)
+#         resolved_merges_file = os.path.join(cache_dir, merges_file)
+#         resolved_sp_file = os.path.join(cache_dir, sp_file)
+#         if tokenizer_class == "wp":
+#             return cls._from_pretrained_wp(resolved_vocab_file, tokenizer_class, *inputs, **kwargs)
+#         elif tokenizer_class == "bpe":
+#             return cls._from_pretrained(resolved_vocab_file, resolved_merges_file, tokenizer_class, *inputs, **kwargs)
+#         elif tokenizer_class == "sp":
+#             return get_encoder(resolved_sp_file, "")
+#
+#     def _from_pretrained_wp(self, vocab_file=None, do_basic_tokenize=True,
+#                             do_lower_case=True, max_len=None,
+#                             never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")):
+#         if not os.path.isfile(vocab_file):
+#             raise ValueError(
+#                 "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained "
+#                 "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
+#                 .format(vocab_file))
+#         self.vocab = load_vocab(vocab_file)
+#         self.ids_to_tokens = collections.OrderedDict([
+#             (ids, tok) for tok, ids in self.vocab.items()
+#         ])
+#         self.do_basic_tokenize = do_basic_tokenize
+#         if do_basic_tokenize:
+#             self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case,
+#                                                   never_split=never_split)
+#         self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+#         self.max_len = max_len if max_len is not None else int(1e12)
+#         self.tokenizer_class = "wp"
+#
+#     def set_special_tokens(self, special_tokens):
+#         """ Add a list of additional tokens to the encoder.
+#             The additional tokens are indexed starting from the last index of the
+#             current vocabulary in the order of the `special_tokens` list.
+#         """
+#         if not special_tokens:
+#             self.special_tokens = {}
+#             self.special_tokens_decoder = {}
+#             return
+#         self.special_tokens = dict((tok, len(self.encoder) + i)
+#                                    for i, tok in enumerate(special_tokens))
+#         self.special_tokens_decoder = {
+#             v: k
+#             for k, v in self.special_tokens.items()
+#         }
+#         logger.info("Special tokens {}".format(self.special_tokens))
+#
+#     def _from_pretrained_bpe(self,
+#                  vocab_file,
+#                  merges_file,
+#                  errors='replace',
+#                  special_tokens=None,
+#                  max_len=None):
+#         self.max_len = max_len if max_len is not None else int(1e12)
+#         self.encoder = json.load(open(vocab_file))
+#         self.decoder = {v: k for k, v in self.encoder.items()}
+#         self.errors = errors  # how to handle errors in decoding
+#         self.byte_encoder = bytes_to_unicode()
+#         self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+#         bpe_data = open(merges_file, encoding='utf-8').read().split('\n')[1:-1]
+#         bpe_merges = [tuple(merge.split()) for merge in bpe_data]
+#         self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+#         self.cache = {}
+#
+#         # Should haved added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
+#         self.pat = re.compile(
+#             r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+#         )
+#
+#         self.special_tokens = {}
+#         self.special_tokens_decoder = {}
+#         self.set_special_tokens(special_tokens)
+#         self.tokenizer_class = "bpe"
+#
+#
+#     def tokenize(self, text):
+#         if self.do_basic_tokenize:
+#             split_tokens = []
+#             for token in self.basic_tokenizer.tokenize(text):
+#                 for sub_token in self.wordpiece_tokenizer.tokenize(token):
+#                     split_tokens.append(sub_token)
+#         else:
+#             split_tokens = self.wordpiece_tokenizer.tokenize(text)
+#         return split_tokens
+#
+#     def convert_tokens_to_ids(self, tokens):
+#         """Converts a sequence of tokens into ids using the vocab."""
+#         ids = []
+#         for token in tokens:
+#             ids.append(self.vocab[token])
+#         if len(ids) > self.max_len:
+#             logger.warning(
+#                 "Token indices sequence length is longer than the specified maximum "
+#                 " sequence length for this BERT model ({} > {}). Running this"
+#                 " sequence through BERT will result in indexing errors".format(
+#                     len(ids), self.max_len))
+#         return ids
+#
+#     def convert_ids_to_tokens(self, ids):
+#         """Converts a sequence of ids in wordpiece tokens using the vocab."""
+#         tokens = []
+#         for i in ids:
+#             tokens.append(self.ids_to_tokens[i])
+#         return tokens
+
+
+
diff --git a/flagai/model/bert_model.py b/flagai/model/bert_model.py
index e91d2c77..5f1da64e 100644
--- a/flagai/model/bert_model.py
+++ b/flagai/model/bert_model.py
@@ -149,10 +149,10 @@ def forward(self,
         # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
         # this attention mask is more simple than the triangular masking of causal attention
         # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
-        extended_attention_mask = extended_attention_mask.unsqueeze(
-            1).unsqueeze(2)
         if attention_mask is not None:
             extended_attention_mask = extended_attention_mask * attention_mask
+        # extended_attention_mask = extended_attention_mask.unsqueeze(
+        #     1).unsqueeze(2)
         # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
         # masked positions, this operation will create a tensor which is 0.0 for
         # positions we want to attend and -10000.0 for masked positions.
@@ -356,7 +356,6 @@ def forward(self, **data):
         input_shape = input_ids.shape
         seq_len = input_shape[1]
         a_mask = self.make_unilm_mask(token_type_ids, seq_len)
-
         encoder_out, pooler_out = self.model(
             input_ids,
             token_type_ids,
diff --git a/flagai/model/mm/__init__.py b/flagai/model/mm/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/flagai/model/mm/clip_model.py b/flagai/model/mm/clip_model.py
new file mode 100644
index 00000000..05a2254b
--- /dev/null
+++ b/flagai/model/mm/clip_model.py
@@ -0,0 +1,303 @@
+""" CLIP Model
+
+Adapted from https://github.com/openai/CLIP. Originally MIT License, Copyright (c) 2021 OpenAI.
+"""
+from collections import OrderedDict
+from dataclasses import dataclass
+from typing import Tuple, Union, Callable, Optional
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+import os 
+if os.getenv('ENV_TYPE') == 'deepspeed':
+    from deepspeed.runtime.activation_checkpointing.checkpointing import checkpoint
+elif os.getenv('ENV_TYPE') == 'pytorch' or os.getenv('ENV_TYPE') == 'pytorchDDP':
+    from torch.utils.checkpoint import checkpoint
+else :
+    print(f"not support the {os.getenv('ENV_TYPE')} for checkpoint activation")
+
+from .utils import to_2tuple
+from flagai.model.base_model import BaseModel
+from ..layers.activations import QuickGELUActivation # NOTE This is slower than nn.GELU or nn.SiLU and uses more GPU memory
+# contrastive loss function, adapted from
+# https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/CLIP.html
+def contrastive_loss(logits: torch.Tensor) -> torch.Tensor:
+    return nn.functional.cross_entropy(logits, torch.arange(len(logits), device=logits.device))
+
+def clip_loss(similarity: torch.Tensor) -> torch.Tensor:
+    caption_loss = contrastive_loss(similarity)
+    image_loss = contrastive_loss(similarity.T)
+    return (caption_loss + image_loss) / 2.0
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        x = F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)
+        return x.to(orig_type)
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, d_model: int, n_head: int, mlp_ratio: float = 4.0, act_layer: Callable = nn.GELU):
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        mlp_width = int(d_model * mlp_ratio)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, mlp_width)),
+            ("gelu", act_layer()),
+            ("c_proj", nn.Linear(mlp_width, d_model))
+        ]))
+        self.ln_2 = LayerNorm(d_model)
+
+    def attention(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None):
+        return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask)[0]
+
+    def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None):
+        x = x + self.attention(self.ln_1(x), attn_mask=attn_mask)
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+
+class Transformer(nn.Module):
+    def __init__(self, width: int, layers: int, heads: int,  mlp_ratio: float = 4.0, act_layer: Callable = nn.GELU):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.grad_checkpointing = False
+
+        self.resblocks = nn.ModuleList([
+            ResidualAttentionBlock(width, heads, mlp_ratio, act_layer=act_layer)
+            for _ in range(layers)
+        ])
+
+    def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None):
+        for r in self.resblocks:
+            if self.grad_checkpointing and not torch.jit.is_scripting():
+                x = checkpoint(r, x, attn_mask)
+            else:
+                x = r(x, attn_mask=attn_mask)
+        return x
+
+
+class VisualTransformer(nn.Module):
+    def __init__(
+            self, image_size: int, patch_size: int, width: int, layers: int, heads: int, mlp_ratio: float,
+            output_dim: int, act_layer: Callable = nn.GELU):
+        super().__init__()
+        self.image_size = to_2tuple(image_size)
+        self.patch_size = to_2tuple(patch_size)
+        self.grid_size = (self.image_size[0] // self.patch_size[0], self.image_size[1] // self.patch_size[1])
+        self.output_dim = output_dim
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
+
+        scale = width ** -0.5
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+        self.positional_embedding = nn.Parameter(scale * torch.randn(self.grid_size[0] * self.grid_size[1] + 1, width))
+        self.ln_pre = LayerNorm(width)
+
+        self.transformer = Transformer(width, layers, heads, mlp_ratio, act_layer=act_layer)
+
+        self.ln_post = LayerNorm(width)
+        self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
+
+    def lock(self, unlocked_groups=0, freeze_bn_stats=False):
+        assert unlocked_groups == 0, 'partial locking not currently supported for this model'
+        for param in self.parameters():
+            param.requires_grad = False
+
+    @torch.jit.ignore
+    def set_grad_checkpointing(self, enable=True):
+        self.transformer.grad_checkpointing = enable
+
+    def forward(self, x: torch.Tensor):
+        x = self.conv1(x)  # shape = [*, width, grid, grid]
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+        x = torch.cat(
+            [self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device),
+             x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+        x = x + self.positional_embedding.to(x.dtype)
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        x = self.ln_post(x[:, 0, :])
+
+        if self.proj is not None:
+            x = x @ self.proj
+
+        return x
+
+
+@dataclass
+class CLIPVisionCfg:
+    layers: Union[Tuple[int, int, int, int], int] = 12
+    width: int = 768
+    head_width: int = 64
+    mlp_ratio: float = 4.0
+    patch_size: int = 16
+    image_size: Union[Tuple[int, int], int] = 224
+    timm_model_name: str = None  # a valid model name overrides layers, width, patch_size
+    timm_model_pretrained: bool = False  # use (imagenet) pretrained weights for named model
+    timm_pool: str = 'avg'  # feature pooling for timm model ('abs_attn', 'rot_attn', 'avg', '')
+    timm_proj: str = 'linear'  # linear projection for timm model output ('linear', 'mlp', '')
+    checkpoint_activations: bool = False
+
+
+@dataclass
+class CLIPTextCfg:
+    context_length: int = 77
+    vocab_size: int = 49408
+    width: int = 512
+    heads: int = 8
+    layers: int = 12
+    checkpoint_activations: bool = False
+
+class CLIP(BaseModel):
+    def __init__(self, config, **kwargs):
+        """
+        embed_dim: int,
+            vision_cfg: CLIPVisionCfg,
+            text_cfg: CLIPTextCfg,
+            quick_gelu: bool = False
+        """
+        super(CLIP,self).__init__(config, **kwargs)
+        embed_dim = config['embed_dim']
+        vision_cfg = config['vision_cfg']
+        text_cfg = config['text_cfg']
+        quick_gelu = config.get('quick_gelu',False)
+        if isinstance(vision_cfg, dict):
+            vision_cfg = CLIPVisionCfg(**vision_cfg)
+        if isinstance(text_cfg, dict):
+            text_cfg = CLIPTextCfg(**text_cfg)
+
+        self.context_length = text_cfg.context_length
+
+        # OpenAI models are pretrained w/ QuickGELU but native nn.GELU is both faster and more
+        # memory efficient in recent PyTorch releases (>= 1.10).
+        # NOTE: timm models always use native GELU regardless of quick_gelu flag.
+        act_layer = QuickGELUActivation if quick_gelu else nn.GELU
+        
+        vision_heads = vision_cfg.width // vision_cfg.head_width
+        self.visual = VisualTransformer(
+            image_size=vision_cfg.image_size,
+            patch_size=vision_cfg.patch_size,
+            width=vision_cfg.width,
+            layers=vision_cfg.layers,
+            heads=vision_heads,
+            mlp_ratio=vision_cfg.mlp_ratio,
+            output_dim=embed_dim,
+            act_layer=act_layer,
+        )
+
+        self.transformer = Transformer(
+            width=text_cfg.width,
+            layers=text_cfg.layers,
+            heads=text_cfg.heads,
+            act_layer=act_layer,
+        )
+        self.image_size = vision_cfg.image_size
+        self.vocab_size = text_cfg.vocab_size
+        self.token_embedding = nn.Embedding(text_cfg.vocab_size, text_cfg.width)
+        self.positional_embedding = nn.Parameter(torch.empty(self.context_length, text_cfg.width))
+        self.ln_final = LayerNorm(text_cfg.width)
+
+        self.text_projection = nn.Parameter(torch.empty(text_cfg.width, embed_dim))
+        self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
+        self.register_buffer('attn_mask', self.build_attention_mask(), persistent=False)
+
+        self.init_parameters()
+
+    def init_parameters(self):
+        nn.init.normal_(self.token_embedding.weight, std=0.02)
+        nn.init.normal_(self.positional_embedding, std=0.01)
+        nn.init.constant_(self.logit_scale, np.log(1 / 0.07))
+
+        if hasattr(self.visual, 'init_parameters'):
+            self.visual.init_parameters()
+
+        proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
+        attn_std = self.transformer.width ** -0.5
+        fc_std = (2 * self.transformer.width) ** -0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+        if self.text_projection is not None:
+            nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
+
+    def build_attention_mask(self):
+        # lazily create causal attention mask, with full attention between the vision tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.empty(self.context_length, self.context_length)
+        mask.fill_(float("-inf"))
+        mask.triu_(1)  # zero out the lower diagonal
+        return mask
+
+    def lock_image_tower(self, unlocked_groups=0, freeze_bn_stats=False):
+        # lock image tower as per LiT - https://arxiv.org/abs/2111.07991
+        self.visual.lock(unlocked_groups=unlocked_groups, freeze_bn_stats=freeze_bn_stats)
+
+    @torch.jit.ignore
+    def set_grad_checkpointing(self, enable=True):
+        self.visual.set_grad_checkpointing(enable)
+        self.transformer.grad_checkpointing = enable
+
+    def encode_image(self, image):
+        if image is not None:
+            # support fp16
+            if next(self.parameters())[0].dtype == torch.float16:
+                if image.dtype == torch.float32:
+                    image = image.half()
+        return self.visual(image)
+
+    def encode_text(self, text):
+        x = self.token_embedding(text)  # [batch_size, n_ctx, d_model]
+
+        x = x + self.positional_embedding
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x, attn_mask=self.attn_mask)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+        x = self.ln_final(x)
+
+        # x.shape = [batch_size, n_ctx, transformer.width]
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection
+
+        return x
+
+    def forward(self, **data):
+        if "image" not in data:
+            return {"logits": self.encode_text(data["text"])}
+        elif "text" not in data:
+            return {"logits": self.encode_image(data["image"])}
+        image_features = self.encode_image(data["image"])
+        image_features = F.normalize(image_features, dim=-1)
+        text_features = self.encode_text(data["text"])
+        text_features = F.normalize(text_features, dim=-1)
+        loss = self.compute_loss(image_features, text_features)
+        logits = torch.cat([image_features, text_features], dim=1)
+
+        return {"logits": logits,
+                "loss": loss}
+
+    def compute_loss(self, image_features, text_features):
+        # cosine similarity as logits
+        logit_scale = self.logit_scale.exp()
+        logits_per_text = torch.matmul(text_features, image_features.t()) * logit_scale
+        logits_per_image = logits_per_text.T
+        loss = clip_loss(logits_per_image)
+        return loss
+
+    def load_weights(self, checkpoint_path):
+        self.load_state_dict(torch.load(checkpoint_path,
+                                        map_location="cpu"),
+                             strict=False)
diff --git a/flagai/model/mm/utils.py b/flagai/model/mm/utils.py
new file mode 100644
index 00000000..bba444f2
--- /dev/null
+++ b/flagai/model/mm/utils.py
@@ -0,0 +1,13 @@
+
+from itertools import repeat
+import collections.abc
+# From PyTorch internals
+def _ntuple(n):
+    def parse(x):
+        if isinstance(x, collections.abc.Iterable):
+            return x
+        return tuple(repeat(x, n))
+    return parse
+
+
+to_2tuple = _ntuple(2) #for clip
\ No newline at end of file
diff --git a/flagai/model/predictor/gpt.py b/flagai/model/predictor/gpt.py
index e99d11f4..f07faab5 100644
--- a/flagai/model/predictor/gpt.py
+++ b/flagai/model/predictor/gpt.py
@@ -7,7 +7,7 @@ def gpt_random_sample_use_cache(model, tokenizer, text, input_max_length, out_ma
                       top_k, top_p, repetition_penalty, temperature, device):
     tokenizer_out = tokenizer.encode_plus(text, max_length=input_max_length)
     token_ids = tokenizer_out["input_ids"]
-    token_end_id = tokenizer.token_end_id
+    token_end_id = tokenizer.get_command_id('eos')
     if token_ids[-1] == token_end_id:
         token_ids = token_ids[:-1]
 
@@ -22,13 +22,13 @@ def gpt_random_sample_use_cache(model, tokenizer, text, input_max_length, out_ma
     token_ids = torch.tensor(token_ids, device=device,
                              dtype=torch.long).view(1, -1)
     output_ids = []
-    sep_id = tokenizer.token_end_id
+    sep_id = tokenizer.get_command_id('eos')
     outputs = model(**{"input_ids": token_ids, "use_cache": True})
     scores = outputs["logits"]
     past_key_values = outputs["hidden_states"]
 
     logit_score = torch.log_softmax(scores[:, -1], dim=-1)
-    logit_score[:, tokenizer.token_unk_id] = -float('Inf')
+    logit_score[:, tokenizer.get_command_id('unk')] = -float('Inf')
 
     filtered_logits = list_processor(token_ids, logit_score)
     next_token = torch.multinomial(F.softmax(filtered_logits, dim=-1),
@@ -42,7 +42,7 @@ def gpt_random_sample_use_cache(model, tokenizer, text, input_max_length, out_ma
             past_key_values = outputs["hidden_states"]
 
             logit_score = torch.log_softmax(scores[:, -1], dim=-1)
-            logit_score[:, tokenizer.token_unk_id] = -float('Inf')
+            logit_score[:, tokenizer.get_command_id('unk')] = -float('Inf')
 
             filtered_logits = list_processor(token_ids, logit_score)
             next_token = torch.multinomial(F.softmax(filtered_logits, dim=-1),
diff --git a/flagai/model/predictor/predictor.py b/flagai/model/predictor/predictor.py
index fa426945..970c1387 100644
--- a/flagai/model/predictor/predictor.py
+++ b/flagai/model/predictor/predictor.py
@@ -168,9 +168,10 @@ def predict_ner(self,
         model.eval()
         device = next(model.parameters()).device
         tokenizer = self.tokenizer
-        tokens = tokenizer.tokenize(text,
-                                    maxlen=maxlen,
-                                    add_spatial_tokens=True)
+        tokens = tokenizer.tokenize(text)
+                                    # maxlen=maxlen,
+                                    # add_spatial_tokens=True)
+        
         mapping = tokenizer.rematch(text, tokens)
         token_ids = tokenizer.convert_tokens_to_ids(tokens)
         token_ids = torch.tensor([token_ids], dtype=torch.long, device=device)
diff --git a/flagai/model/predictor/utils.py b/flagai/model/predictor/utils.py
index 6ff3ba86..e7184e61 100644
--- a/flagai/model/predictor/utils.py
+++ b/flagai/model/predictor/utils.py
@@ -468,7 +468,7 @@ def t5_random_sample(model, tokenizer, text, input_max_length, out_max_length,
     token_ids = torch.tensor(token_ids, device=device,
                              dtype=torch.long).view(1, -1)
     output_ids = []
-    input_decoder_ids = torch.tensor(tokenizer.token_start_id,
+    input_decoder_ids = torch.tensor(tokenizer.get_command_id('cls'),
                                      device=device,
                                      dtype=torch.long).view(1, -1)
     lp = [
@@ -485,13 +485,13 @@ def t5_random_sample(model, tokenizer, text, input_max_length, out_max_length,
                 "decoder_input_ids": input_decoder_ids
             })["logits"]
             logit_score = torch.log_softmax(scores[:, -1], dim=-1)
-            logit_score[:, tokenizer.token_unk_id] = -float('Inf')
+            logit_score[:, tokenizer.get_command_id('unk')] = -float('Inf')
             # filtered_logits = top_k_top_p_filtering(logit_score, top_k=top_k, top_p=top_p)
             filtered_logits = list_processor(input_decoder_ids, logit_score)
 
             filterd_logits_prob = F.softmax(filtered_logits, dim=-1)
             next_token = torch.multinomial(filterd_logits_prob, num_samples=1)
-            if tokenizer.token_end_id == next_token.item():
+            if tokenizer.get_command_id('eos') == next_token.item():
                 break
             output_ids.append(next_token.item())
             input_decoder_ids = torch.cat(
@@ -526,12 +526,12 @@ def bert_random_sample(model, tokenizer, text, input_max_length,
                 "segment_ids": token_type_ids
             })["logits"]
             logit_score = torch.log_softmax(scores[:, -1], dim=-1)
-            logit_score[:, tokenizer.token_unk_id] = -float('Inf')
+            logit_score[:, tokenizer.get_command_id('unk')] = -float('Inf')
             filtered_logits = list_processor(token_ids, logit_score)
 
             filterd_logits_prob = F.softmax(filtered_logits, dim=-1)
             next_token = torch.multinomial(filterd_logits_prob, num_samples=1)
-            if tokenizer.token_end_id == next_token.item():
+            if tokenizer.get_command_id('eos') == next_token.item():
                 break
             output_ids.append(next_token.item())
             token_ids = torch.cat((token_ids, next_token.long()), dim=1)
@@ -546,7 +546,7 @@ def gpt_random_sample(model, tokenizer, text, input_max_length, out_max_length,
                       top_k, top_p, repetition_penalty, temperature, device):
     tokenizer_out = tokenizer.encode_plus(text, max_length=input_max_length)
     token_ids = tokenizer_out["input_ids"]
-    token_end_id = tokenizer.token_end_id
+    token_end_id = tokenizer.get_command_id('eos')
     if token_ids[-1] == token_end_id:
         token_ids = token_ids[:-1]
 
@@ -561,12 +561,12 @@ def gpt_random_sample(model, tokenizer, text, input_max_length, out_max_length,
     token_ids = torch.tensor(token_ids, device=device,
                              dtype=torch.long).view(1, -1)
     output_ids = []
-    sep_id = tokenizer.token_end_id
+    sep_id = tokenizer.get_command_id('eos')
     with torch.no_grad():
         for step in range(out_max_length):
             scores = model(**{"input_ids": token_ids})["logits"]
             logit_score = torch.log_softmax(scores[:, -1], dim=-1)
-            logit_score[:, tokenizer.token_unk_id] = -float('Inf')
+            logit_score[:, tokenizer.get_command_id('unk')] = -float('Inf')
 
             filtered_logits = list_processor(token_ids, logit_score)
             next_token = torch.multinomial(F.softmax(filtered_logits, dim=-1),
@@ -670,7 +670,7 @@ def glm_beamsearch(model, tokenizer, text, out_max_length, beam_size):  #
 def bert_beamsearch(model, tokenizer, text, input_max_length, out_max_length,
                     beam_size):
     tokenizer_out = tokenizer.encode_plus(text, max_length=input_max_length)
-    vocab = tokenizer.vocab
+    vocab = tokenizer.get_vocab()
     token_ids = tokenizer_out["input_ids"]
     token_ids = np.array(token_ids).reshape(1, -1)
     out_puts_ids = bert_beam_search(model,
@@ -752,8 +752,8 @@ def t5_beam_search(model,
                    beam_size=1,
                    out_max_length=50):
 
-    sep_id = tokenizer.token_end_id
-    decoder_input_ids = np.array(tokenizer.token_start_id,
+    sep_id = tokenizer.get_command_id('eos')
+    decoder_input_ids = np.array(tokenizer.get_command_id('cls'),
                                  dtype=np.int64).reshape(1, -1)
 
     output_ids = None
@@ -824,7 +824,7 @@ def glm_sample_sequence(model,
                         out_seq_length=512,
                         temperature=0.9,
                         top_k=40):
-    tokens = context_tokens.new_full((1, 1), tokenizer.get_command('sop').Id)
+    tokens = context_tokens.new_full((1, 1), tokenizer.get_command_id('sop'))
     counter = 0
     if mems is None:
         mems = []
@@ -879,9 +879,9 @@ def glm_generate_sample(
     if 'MASK]' not in text:
         text += ' ' + generation_mask
     context_tokens = tokenizer.EncodeAsIds(text)
-    context_tokens = [tokenizer.get_command('ENC').Id] + context_tokens
+    context_tokens = [tokenizer.get_command_id('cls')] + context_tokens
     if not text.endswith('[gMASK]'):
-        context_tokens = context_tokens + [tokenizer.get_command('eos').Id]
+        context_tokens = context_tokens + [tokenizer.get_command_id('eos')]
     context_length = len(context_tokens)
     context_length_tensor = torch.cuda.LongTensor([context_length])
     context_length = context_length_tensor[0].item()
@@ -905,8 +905,8 @@ def glm_generate_sample(
     position_ids = torch.stack((position_ids, block_position_ids), dim=0)
     position_ids = position_ids.unsqueeze(0)
     mask_tokens = ['MASK', 'sMASK', 'gMASK']
-    mask_tokens = [tokenizer.get_command(token).Id for token in mask_tokens]
-    end_tokens = [tokenizer.get_command('eop').Id, eod_token]
+    mask_tokens = [tokenizer.get_command_id(token) for token in mask_tokens]
+    end_tokens = [tokenizer.get_command_id('eop'), eod_token]
     mask_positions = []
     for token in mask_tokens:
         mask_positions += (context_tokens_tensor == token).nonzero(
@@ -938,7 +938,7 @@ def gpt_beam_search(model,
                     beam_size=1,
                     out_max_length=50):
 
-    sep_id = tokenizer.token_end_id
+    sep_id = tokenizer.get_command_id('eos')
 
     output_ids = None
     with torch.no_grad():
diff --git a/flagai/model/utils.py b/flagai/model/utils.py
index ab3ba441..0244e6ca 100644
--- a/flagai/model/utils.py
+++ b/flagai/model/utils.py
@@ -70,3 +70,16 @@ def init_(tensor):
         return torch.nn.init.normal_(tensor, mean=mean, std=std)
 
     return init_
+    
+from itertools import repeat
+import collections.abc
+# From PyTorch internals
+def _ntuple(n):
+    def parse(x):
+        if isinstance(x, collections.abc.Iterable):
+            return x
+        return tuple(repeat(x, n))
+    return parse
+
+
+to_2tuple = _ntuple(2) #for clip 
\ No newline at end of file
diff --git a/flagai/model/vision/swinv1.py b/flagai/model/vision/swinv1.py
new file mode 100644
index 00000000..d2ccb0d3
--- /dev/null
+++ b/flagai/model/vision/swinv1.py
@@ -0,0 +1,647 @@
+"""
+# Copyright © 2022 BAAI. All rights reserved.
+"""
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint as checkpoint
+from flagai.model.vision.layers import DropPath, to_2tuple, trunc_normal_
+import os
+if os.getenv('ENV_TYPE') == 'deepspeed':
+    from deepspeed.runtime.activation_checkpointing.checkpointing import checkpoint
+else:
+    from torch.utils.checkpoint import checkpoint
+try:
+    import os, sys
+    kernel_path = os.path.abspath(os.path.join('..'))
+    sys.path.append(kernel_path)
+    from kernels.window_process.window_process import WindowProcess, WindowProcessReverse
+
+except:
+    WindowProcess = None
+    WindowProcessReverse = None
+    print("[Warning] Fused window process have not been installed. Please refer to get_started.md for installation. https://github.com/microsoft/Swin-Transformer")
+
+class SwinTransformerConfig:
+    def __init__(self, img_size=224, patch_size=4, in_chans=3,
+                 embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32],
+                 window_size=7, mlp_ratio=4., qkv_bias=True,qk_scale=None,
+                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1,
+                 norm_layer=nn.LayerNorm, ape=False, patch_norm=True,
+                 use_checkpoint=False, pretrained_window_sizes=[0, 0, 0, 0],checkpoint_activations=False,
+                 fused_window_process=False,num_classes = 1000):
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+        self.depths = depths
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.mlp_ratio = mlp_ratio
+        self.qkv_bias = qkv_bias
+        self.qk_scale = qk_scale
+        self.drop_rate = drop_rate
+        self.attn_drop_rate = attn_drop_rate
+        self.drop_path_rate = drop_path_rate
+        self.norm_layer = norm_layer
+        self.ape = ape
+        self.patch_norm = patch_norm
+        self.pretrained_window_sizes = pretrained_window_sizes
+        self.fused_window_process = fused_window_process
+        self.checkpoint_activations = checkpoint_activations
+        self.num_classes = num_classes
+
+class Mlp(nn.Module):
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+def window_partition(x, window_size):
+    """
+    Args:
+        x: (B, H, W, C)
+        window_size (int): window size
+
+    Returns:
+        windows: (num_windows*B, window_size, window_size, C)
+    """
+    B, H, W, C = x.shape
+    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)
+    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+    return windows
+
+def window_reverse(windows, window_size, H, W):
+    """
+    Args:
+        windows: (num_windows*B, window_size, window_size, C)
+        window_size (int): Window size
+        H (int): Height of image
+        W (int): Width of image
+
+    Returns:
+        x: (B, H, W, C)
+    """
+    B = int(windows.shape[0] / (H * W / window_size / window_size))
+    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
+    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
+    return x
+
+class WindowAttention(nn.Module):
+    r""" Window based multi-head self attention (W-MSA) module with relative position bias.
+    It supports both of shifted and non-shifted window.
+
+    Args:
+        dim (int): Number of input channels.
+        window_size (tuple[int]): The height and width of the window.
+        num_heads (int): Number of attention heads.
+        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set
+        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
+        proj_drop (float, optional): Dropout ratio of output. Default: 0.0
+    """
+
+    def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.):
+
+        super().__init__()
+        self.dim = dim
+        self.window_size = window_size  # Wh, Ww
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+
+        # define a parameter table of relative position bias
+        self.relative_position_bias_table = nn.Parameter(
+            torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads))  # 2*Wh-1 * 2*Ww-1, nH
+
+        # get pair-wise relative position index for each token inside the window
+        coords_h = torch.arange(self.window_size[0])
+        coords_w = torch.arange(self.window_size[1])
+        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
+        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += self.window_size[1] - 1
+        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
+        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+        self.register_buffer("relative_position_index", relative_position_index)
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+        trunc_normal_(self.relative_position_bias_table, std=.02)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x, mask=None):
+        """
+        Args:
+            x: input features with shape of (num_windows*B, N, C)
+            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
+        """
+        B_, N, C = x.shape
+        qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
+
+        q = q * self.scale
+        attn = (q @ k.transpose(-2, -1))
+
+        relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
+            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1)  # Wh*Ww,Wh*Ww,nH
+        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
+        attn = attn + relative_position_bias.unsqueeze(0)
+
+        if mask is not None:
+            nW = mask.shape[0]
+            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)
+            attn = attn.view(-1, self.num_heads, N, N)
+            attn = self.softmax(attn)
+        else:
+            attn = self.softmax(attn)
+
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+    def extra_repr(self) -> str:
+        return f'dim={self.dim}, window_size={self.window_size}, num_heads={self.num_heads}'
+
+    def flops(self, N):
+        # calculate flops for 1 window with token length of N
+        flops = 0
+        # qkv = self.qkv(x)
+        flops += N * self.dim * 3 * self.dim
+        # attn = (q @ k.transpose(-2, -1))
+        flops += self.num_heads * N * (self.dim // self.num_heads) * N
+        #  x = (attn @ v)
+        flops += self.num_heads * N * N * (self.dim // self.num_heads)
+        # x = self.proj(x)
+        flops += N * self.dim * self.dim
+        return flops
+
+
+class SwinTransformerBlock(nn.Module):
+    r""" Swin Transformer Block.
+
+    Args:
+        dim (int): Number of input channels.
+        input_resolution (tuple[int]): Input resulotion.
+        num_heads (int): Number of attention heads.
+        window_size (int): Window size.
+        shift_size (int): Shift size for SW-MSA.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float, optional): Stochastic depth rate. Default: 0.0
+        act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+        fused_window_process (bool, optional): If True, use one kernel to fused window shift & window partition for acceleration, similar for the reversed part. Default: False
+    """
+
+    def __init__(self, dim, input_resolution, num_heads, window_size=7, shift_size=0,
+                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0.,
+                 act_layer=nn.GELU, norm_layer=nn.LayerNorm,
+                 fused_window_process=False):
+        super().__init__()
+        self.dim = dim
+        self.input_resolution = input_resolution
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.shift_size = shift_size
+        self.mlp_ratio = mlp_ratio
+        if min(self.input_resolution) <= self.window_size:
+            # if window size is larger than input resolution, we don't partition windows
+            self.shift_size = 0
+            self.window_size = min(self.input_resolution)
+        assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size"
+
+        self.norm1 = norm_layer(dim)
+        self.attn = WindowAttention(
+            dim, window_size=to_2tuple(self.window_size), num_heads=num_heads,
+            qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+
+        if self.shift_size > 0:
+            # calculate attention mask for SW-MSA
+            H, W = self.input_resolution
+            img_mask = torch.zeros((1, H, W, 1))  # 1 H W 1
+            h_slices = (slice(0, -self.window_size),
+                        slice(-self.window_size, -self.shift_size),
+                        slice(-self.shift_size, None))
+            w_slices = (slice(0, -self.window_size),
+                        slice(-self.window_size, -self.shift_size),
+                        slice(-self.shift_size, None))
+            cnt = 0
+            for h in h_slices:
+                for w in w_slices:
+                    img_mask[:, h, w, :] = cnt
+                    cnt += 1
+
+            mask_windows = window_partition(img_mask, self.window_size)  # nW, window_size, window_size, 1
+            mask_windows = mask_windows.view(-1, self.window_size * self.window_size)
+            attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
+            attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
+        else:
+            attn_mask = None
+
+        self.register_buffer("attn_mask", attn_mask)
+        self.fused_window_process = fused_window_process
+
+    def forward(self, x):
+        H, W = self.input_resolution
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+
+        shortcut = x
+        x = self.norm1(x)
+        x = x.view(B, H, W, C)
+
+        # cyclic shift
+        if self.shift_size > 0:
+            if not self.fused_window_process:
+                shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
+                # partition windows
+                x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C
+            else:
+                x_windows = WindowProcess.apply(x, B, H, W, C, -self.shift_size, self.window_size)
+        else:
+            shifted_x = x
+            # partition windows
+            x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C
+
+        x_windows = x_windows.view(-1, self.window_size * self.window_size, C)  # nW*B, window_size*window_size, C
+
+        # W-MSA/SW-MSA
+        attn_windows = self.attn(x_windows, mask=self.attn_mask)  # nW*B, window_size*window_size, C
+
+        # merge windows
+        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
+
+        # reverse cyclic shift
+        if self.shift_size > 0:
+            if not self.fused_window_process:
+                shifted_x = window_reverse(attn_windows, self.window_size, H, W)  # B H' W' C
+                x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
+            else:
+                x = WindowProcessReverse.apply(attn_windows, B, H, W, C, self.shift_size, self.window_size)
+        else:
+            shifted_x = window_reverse(attn_windows, self.window_size, H, W)  # B H' W' C
+            x = shifted_x
+        x = x.view(B, H * W, C)
+        x = shortcut + self.drop_path(x)
+
+        # FFN
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+
+        return x
+
+    def extra_repr(self) -> str:
+        return f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " \
+               f"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}"
+
+    def flops(self):
+        flops = 0
+        H, W = self.input_resolution
+        # norm1
+        flops += self.dim * H * W
+        # W-MSA/SW-MSA
+        nW = H * W / self.window_size / self.window_size
+        flops += nW * self.attn.flops(self.window_size * self.window_size)
+        # mlp
+        flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio
+        # norm2
+        flops += self.dim * H * W
+        return flops
+
+
+class PatchMerging(nn.Module):
+    r""" Patch Merging Layer.
+
+    Args:
+        input_resolution (tuple[int]): Resolution of input feature.
+        dim (int): Number of input channels.
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+
+    def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.dim = dim
+        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)
+        self.norm = norm_layer(4 * dim)
+
+    def forward(self, x):
+        """
+        x: B, H*W, C
+        """
+        H, W = self.input_resolution
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+        assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even."
+
+        x = x.view(B, H, W, C)
+
+        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C
+        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C
+        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C
+        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C
+        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C
+        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C
+
+        x = self.norm(x)
+        x = self.reduction(x)
+
+        return x
+
+    def extra_repr(self) -> str:
+        return f"input_resolution={self.input_resolution}, dim={self.dim}"
+
+    def flops(self):
+        H, W = self.input_resolution
+        flops = H * W * self.dim
+        flops += (H // 2) * (W // 2) * 4 * self.dim * 2 * self.dim
+        return flops
+
+
+class BasicLayer(nn.Module):
+    """ A basic Swin Transformer layer for one stage.
+
+    Args:
+        dim (int): Number of input channels.
+        input_resolution (tuple[int]): Input resolution.
+        depth (int): Number of blocks.
+        num_heads (int): Number of attention heads.
+        window_size (int): Local window size.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
+        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
+        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+        fused_window_process (bool, optional): If True, use one kernel to fused window shift & window partition for acceleration, similar for the reversed part. Default: False
+    """
+
+    def __init__(self, dim, input_resolution, depth, num_heads, window_size,
+                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0.,
+                 drop_path=0., norm_layer=nn.LayerNorm, downsample=None, use_checkpoint=False,
+                 fused_window_process=False):
+
+        super().__init__()
+        self.dim = dim
+        self.input_resolution = input_resolution
+        self.depth = depth
+        self.use_checkpoint = use_checkpoint
+
+        # build blocks
+        self.blocks = nn.ModuleList([
+            SwinTransformerBlock(dim=dim, input_resolution=input_resolution,
+                                 num_heads=num_heads, window_size=window_size,
+                                 shift_size=0 if (i % 2 == 0) else window_size // 2,
+                                 mlp_ratio=mlp_ratio,
+                                 qkv_bias=qkv_bias, qk_scale=qk_scale,
+                                 drop=drop, attn_drop=attn_drop,
+                                 drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
+                                 norm_layer=norm_layer,
+                                 fused_window_process=fused_window_process)
+            for i in range(depth)])
+
+        # patch merging layer
+        if downsample is not None:
+            self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer)
+        else:
+            self.downsample = None
+
+    def forward(self, x, checkpoint_activations=False):
+        for blk in self.blocks:
+            if self.use_checkpoint or checkpoint_activations:
+                x = checkpoint(blk, x)
+            else:
+                x = blk(x)
+        if self.downsample is not None:
+            x = self.downsample(x)
+        return x
+
+    def extra_repr(self) -> str:
+        return f"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}"
+
+    def flops(self):
+        flops = 0
+        for blk in self.blocks:
+            flops += blk.flops()
+        if self.downsample is not None:
+            flops += self.downsample.flops()
+        return flops
+
+
+class PatchEmbed(nn.Module):
+    r""" Image to Patch Embedding
+
+    Args:
+        img_size (int): Image size.  Default: 224.
+        patch_size (int): Patch token size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        norm_layer (nn.Module, optional): Normalization layer. Default: None
+    """
+
+    def __init__(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):
+        super().__init__()
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        patches_resolution = [img_size[0] // patch_size[0], img_size[1] // patch_size[1]]
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.patches_resolution = patches_resolution
+        self.num_patches = patches_resolution[0] * patches_resolution[1]
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+        if norm_layer is not None:
+            self.norm = norm_layer(embed_dim)
+        else:
+            self.norm = None
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        # FIXME look at relaxing size constraints
+        assert H == self.img_size[0] and W == self.img_size[1], \
+            f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
+        x = self.proj(x).flatten(2).transpose(1, 2)  # B Ph*Pw C
+        if self.norm is not None:
+            x = self.norm(x)
+        return x
+
+    def flops(self):
+        Ho, Wo = self.patches_resolution
+        flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
+        if self.norm is not None:
+            flops += Ho * Wo * self.embed_dim
+        return flops
+
+from flagai.model.base_model import BaseModel
+class SwinTransformer(BaseModel):
+    r""" Swin Transformer
+        A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -
+          https://arxiv.org/pdf/2103.14030
+
+    Args:
+        img_size (int | tuple(int)): Input image size. Default 224
+        patch_size (int | tuple(int)): Patch size. Default: 4
+        in_chans (int): Number of input image channels. Default: 3
+        num_classes (int): Number of classes for classification head. Default: 1000
+        embed_dim (int): Patch embedding dimension. Default: 96
+        depths (tuple(int)): Depth of each Swin Transformer layer.
+        num_heads (tuple(int)): Number of attention heads in different layers.
+        window_size (int): Window size. Default: 7
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4
+        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. Default: None
+        drop_rate (float): Dropout rate. Default: 0
+        attn_drop_rate (float): Attention dropout rate. Default: 0
+        drop_path_rate (float): Stochastic depth rate. Default: 0.1
+        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
+        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False
+        patch_norm (bool): If True, add normalization after patch embedding. Default: True
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False
+        fused_window_process (bool, optional): If True, use one kernel to fused window shift & window partition for acceleration, similar for the reversed part. Default: False
+    """
+
+    def __init__(self, config, num_classes, **kwargs):
+        super().__init__(config, **kwargs)
+        swinv1_config = SwinTransformerConfig(**config)
+
+        self.num_classes = num_classes
+        self.num_layers = len(swinv1_config.depths)
+        self.embed_dim = swinv1_config.embed_dim
+        self.ape = swinv1_config.ape
+        self.patch_norm = swinv1_config.patch_norm
+        self.num_features = int(swinv1_config.embed_dim * 2 ** (self.num_layers - 1))
+        self.mlp_ratio = swinv1_config.mlp_ratio
+
+        # split image into non-overlapping patches
+        self.patch_embed = PatchEmbed(
+            img_size=swinv1_config.img_size, patch_size=swinv1_config.patch_size, in_chans=swinv1_config.in_chans, embed_dim=swinv1_config.embed_dim,
+            norm_layer=swinv1_config.norm_layer if self.patch_norm else None)
+        num_patches = self.patch_embed.num_patches
+        patches_resolution = self.patch_embed.patches_resolution
+        self.patches_resolution = patches_resolution
+
+        # absolute position embedding
+        if self.ape:
+            self.absolute_pos_embed = nn.Parameter(torch.zeros(1, num_patches, swinv1_config.embed_dim))
+            trunc_normal_(self.absolute_pos_embed, std=.02)
+
+        self.pos_drop = nn.Dropout(p=swinv1_config.drop_rate)
+
+        # stochastic depth
+        dpr = [x.item() for x in torch.linspace(0, swinv1_config.drop_path_rate, sum(swinv1_config.depths))]  # stochastic depth decay rule
+
+        # build layers
+        self.layers = nn.ModuleList()
+        for i_layer in range(self.num_layers):
+            layer = BasicLayer(dim=int(swinv1_config.embed_dim * 2 ** i_layer),
+                               input_resolution=(patches_resolution[0] // (2 ** i_layer),
+                                                 patches_resolution[1] // (2 ** i_layer)),
+                               depth=swinv1_config.depths[i_layer],
+                               num_heads=swinv1_config.num_heads[i_layer],
+                               window_size=swinv1_config.window_size,
+                               mlp_ratio=self.mlp_ratio,
+                               qkv_bias=swinv1_config.qkv_bias, qk_scale=swinv1_config.qk_scale,
+                               drop=swinv1_config.drop_rate, attn_drop=swinv1_config.attn_drop_rate,
+                               drop_path=dpr[sum(swinv1_config.depths[:i_layer]):sum(swinv1_config.depths[:i_layer + 1])],
+                               norm_layer=swinv1_config.norm_layer,
+                               downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
+                               use_checkpoint=swinv1_config.checkpoint_activations,
+                               fused_window_process=swinv1_config.fused_window_process)
+            self.layers.append(layer)
+
+        self.norm = swinv1_config.norm_layer(self.num_features)
+        self.avgpool = nn.AdaptiveAvgPool1d(1)
+        self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity()
+
+        self.apply(self._init_weights)
+
+    def load_weights(self, checkpoint_path):
+
+        return self.load_state_dict(torch.load(checkpoint_path))
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {'absolute_pos_embed'}
+
+    @torch.jit.ignore
+    def no_weight_decay_keywords(self):
+        return {'relative_position_bias_table'}
+
+    def forward_features(self, x):
+        x = self.patch_embed(x)
+        if self.ape:
+            x = x + self.absolute_pos_embed
+        x = self.pos_drop(x)
+
+        for layer in self.layers:
+            x = layer(x, checkpoint_activations=self.config["checkpoint_activations"])
+
+        x = self.norm(x)  # B L C
+        x = self.avgpool(x.transpose(1, 2))  # B C 1
+        x = torch.flatten(x, 1)
+        return x
+
+    def compute_loss(self, pred, labels):
+        loss_func = nn.CrossEntropyLoss()
+        loss = loss_func(pred, labels)
+        return loss
+
+    def forward(self, images=None, labels=None, **kwargs):
+        x = self.forward_features(images)
+        x = self.head(x)
+        return_data = {"logits": x}
+        if labels is not None:
+            loss = self.compute_loss(x, labels)
+            return_data["loss"] = loss
+        return return_data
+
+    def flops(self):
+        flops = 0
+        flops += self.patch_embed.flops()
+        for i, layer in enumerate(self.layers):
+            flops += layer.flops()
+        flops += self.num_features * self.patches_resolution[0] * self.patches_resolution[1] // (2 ** self.num_layers)
+        flops += self.num_features * self.num_classes
+        return flops
diff --git a/flagai/model/vision/swinv2.py b/flagai/model/vision/swinv2.py
new file mode 100644
index 00000000..1a58d45a
--- /dev/null
+++ b/flagai/model/vision/swinv2.py
@@ -0,0 +1,686 @@
+"""
+# Copyright © 2022 BAAI. All rights reserved.
+"""
+
+# --------------------------------------------------------
+# Swin Transformer V2
+# Copyright (c) 2022 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Ze Liu
+# --------------------------------------------------------
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint as checkpoint
+from timm.models.layers import DropPath, to_2tuple, trunc_normal_
+import numpy as np
+from flagai.model.base_model import BaseModel
+import os
+if os.getenv('ENV_TYPE') == 'deepspeed':
+    from deepspeed.runtime.activation_checkpointing.checkpointing import checkpoint
+else:
+    from torch.utils.checkpoint import checkpoint
+
+class Mlp(nn.Module):
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+def window_partition(x, window_size):
+    """
+    Args:
+        x: (B, H, W, C)
+        window_size (int): window size
+    Returns:
+        windows: (num_windows*B, window_size, window_size, C)
+    """
+    B, H, W, C = x.shape
+    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)
+    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+    return windows
+
+
+def window_reverse(windows, window_size, H, W):
+    """
+    Args:
+        windows: (num_windows*B, window_size, window_size, C)
+        window_size (int): Window size
+        H (int): Height of image
+        W (int): Width of image
+    Returns:
+        x: (B, H, W, C)
+    """
+    B = int(windows.shape[0] / (H * W / window_size / window_size))
+    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
+    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
+    return x
+
+
+class WindowAttention(nn.Module):
+    r""" Window based multi-head self attention (W-MSA) module with relative position bias.
+    It supports both of shifted and non-shifted window.
+    Args:
+        dim (int): Number of input channels.
+        window_size (tuple[int]): The height and width of the window.
+        num_heads (int): Number of attention heads.
+        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
+        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
+        proj_drop (float, optional): Dropout ratio of output. Default: 0.0
+        pretrained_window_size (tuple[int]): The height and width of the window in pre-training.
+    """
+
+    def __init__(self, dim, window_size, num_heads, qkv_bias=True, attn_drop=0., proj_drop=0.,
+                 pretrained_window_size=[0, 0]):
+
+        super().__init__()
+        self.dim = dim
+        self.window_size = window_size  # Wh, Ww
+        self.pretrained_window_size = pretrained_window_size
+        self.num_heads = num_heads
+
+        self.logit_scale = nn.Parameter(torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True)
+
+        # mlp to generate continuous relative position bias
+        self.cpb_mlp = nn.Sequential(nn.Linear(2, 512, bias=True),
+                                     nn.ReLU(inplace=True),
+                                     nn.Linear(512, num_heads, bias=False))
+
+        # get relative_coords_table
+        relative_coords_h = torch.arange(-(self.window_size[0] - 1), self.window_size[0], dtype=torch.float32)
+        relative_coords_w = torch.arange(-(self.window_size[1] - 1), self.window_size[1], dtype=torch.float32)
+        relative_coords_table = torch.stack(
+            torch.meshgrid([relative_coords_h,
+                            relative_coords_w])).permute(1, 2, 0).contiguous().unsqueeze(0)  # 1, 2*Wh-1, 2*Ww-1, 2
+        if pretrained_window_size[0] > 0:
+            relative_coords_table[:, :, :, 0] /= (pretrained_window_size[0] - 1)
+            relative_coords_table[:, :, :, 1] /= (pretrained_window_size[1] - 1)
+        else:
+            relative_coords_table[:, :, :, 0] /= (self.window_size[0] - 1)
+            relative_coords_table[:, :, :, 1] /= (self.window_size[1] - 1)
+        relative_coords_table *= 8  # normalize to -8, 8
+        relative_coords_table = torch.sign(relative_coords_table) * torch.log2(
+            torch.abs(relative_coords_table) + 1.0) / np.log2(8)
+
+        self.register_buffer("relative_coords_table", relative_coords_table)
+
+        # get pair-wise relative position index for each token inside the window
+        coords_h = torch.arange(self.window_size[0])
+        coords_w = torch.arange(self.window_size[1])
+        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
+        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += self.window_size[1] - 1
+        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
+        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+        self.register_buffer("relative_position_index", relative_position_index)
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=False)
+        if qkv_bias:
+            self.q_bias = nn.Parameter(torch.zeros(dim))
+            self.v_bias = nn.Parameter(torch.zeros(dim))
+        else:
+            self.q_bias = None
+            self.v_bias = None
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x, mask=None):
+        """
+        Args:
+            x: input features with shape of (num_windows*B, N, C)
+            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
+        """
+        B_, N, C = x.shape
+        qkv_bias = None
+        if self.q_bias is not None:
+            qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias))
+        qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias)
+        qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
+
+        # cosine attention
+        attn = (F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1))
+        logit_scale = torch.clamp(self.logit_scale, max=torch.log(torch.tensor(1. / 0.01))).exp()
+        attn = attn * logit_scale
+
+        relative_position_bias_table = self.cpb_mlp(self.relative_coords_table).view(-1, self.num_heads)
+        relative_position_bias = relative_position_bias_table[self.relative_position_index.view(-1)].view(
+            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1)  # Wh*Ww,Wh*Ww,nH
+        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
+        relative_position_bias = 16 * torch.sigmoid(relative_position_bias)
+        attn = attn + relative_position_bias.unsqueeze(0)
+
+        if mask is not None:
+            nW = mask.shape[0]
+            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)
+            attn = attn.view(-1, self.num_heads, N, N)
+            attn = self.softmax(attn)
+        else:
+            attn = self.softmax(attn)
+
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+    def extra_repr(self) -> str:
+        return f'dim={self.dim}, window_size={self.window_size}, ' \
+               f'pretrained_window_size={self.pretrained_window_size}, num_heads={self.num_heads}'
+
+    def flops(self, N):
+        # calculate flops for 1 window with token length of N
+        flops = 0
+        # qkv = self.qkv(x)
+        flops += N * self.dim * 3 * self.dim
+        # attn = (q @ k.transpose(-2, -1))
+        flops += self.num_heads * N * (self.dim // self.num_heads) * N
+        #  x = (attn @ v)
+        flops += self.num_heads * N * N * (self.dim // self.num_heads)
+        # x = self.proj(x)
+        flops += N * self.dim * self.dim
+        return flops
+
+
+class SwinTransformerBlock(nn.Module):
+    r""" Swin Transformer Block.
+    Args:
+        dim (int): Number of input channels.
+        input_resolution (tuple[int]): Input resulotion.
+        num_heads (int): Number of attention heads.
+        window_size (int): Window size.
+        shift_size (int): Shift size for SW-MSA.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float, optional): Stochastic depth rate. Default: 0.0
+        act_layer (nn.Module, optional): Activation layer. Default: nn.GELU
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+        pretrained_window_size (int): Window size in pre-training.
+    """
+
+    def __init__(self, dim, input_resolution, num_heads, window_size=7, shift_size=0,
+                 mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0., drop_path=0.,
+                 act_layer=nn.GELU, norm_layer=nn.LayerNorm, pretrained_window_size=0):
+        super().__init__()
+        self.dim = dim
+        self.input_resolution = input_resolution
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.shift_size = shift_size
+        self.mlp_ratio = mlp_ratio
+        if min(self.input_resolution) <= self.window_size:
+            # if window size is larger than input resolution, we don't partition windows
+            self.shift_size = 0
+            self.window_size = min(self.input_resolution)
+        assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size"
+
+        self.norm1 = norm_layer(dim)
+        self.attn = WindowAttention(
+            dim, window_size=to_2tuple(self.window_size), num_heads=num_heads,
+            qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop,
+            pretrained_window_size=to_2tuple(pretrained_window_size))
+
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
+
+        if self.shift_size > 0:
+            # calculate attention mask for SW-MSA
+            H, W = self.input_resolution
+            img_mask = torch.zeros((1, H, W, 1))  # 1 H W 1
+            h_slices = (slice(0, -self.window_size),
+                        slice(-self.window_size, -self.shift_size),
+                        slice(-self.shift_size, None))
+            w_slices = (slice(0, -self.window_size),
+                        slice(-self.window_size, -self.shift_size),
+                        slice(-self.shift_size, None))
+            cnt = 0
+            for h in h_slices:
+                for w in w_slices:
+                    img_mask[:, h, w, :] = cnt
+                    cnt += 1
+
+            mask_windows = window_partition(img_mask, self.window_size)  # nW, window_size, window_size, 1
+            mask_windows = mask_windows.view(-1, self.window_size * self.window_size)
+            attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
+            attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
+        else:
+            attn_mask = None
+
+        self.register_buffer("attn_mask", attn_mask)
+
+    def forward(self, x):
+        H, W = self.input_resolution
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+
+        shortcut = x
+        x = x.view(B, H, W, C)
+
+        # cyclic shift
+        if self.shift_size > 0:
+            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
+        else:
+            shifted_x = x
+
+        # partition windows
+        x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C
+        x_windows = x_windows.view(-1, self.window_size * self.window_size, C)  # nW*B, window_size*window_size, C
+
+        # W-MSA/SW-MSA
+        attn_windows = self.attn(x_windows, mask=self.attn_mask)  # nW*B, window_size*window_size, C
+
+        # merge windows
+        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
+        shifted_x = window_reverse(attn_windows, self.window_size, H, W)  # B H' W' C
+
+        # reverse cyclic shift
+        if self.shift_size > 0:
+            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
+        else:
+            x = shifted_x
+        x = x.view(B, H * W, C)
+        x = shortcut + self.drop_path(self.norm1(x))
+
+        # FFN
+        x = x + self.drop_path(self.norm2(self.mlp(x)))
+
+        return x
+
+    def extra_repr(self) -> str:
+        return f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " \
+               f"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}"
+
+    def flops(self):
+        flops = 0
+        H, W = self.input_resolution
+        # norm1
+        flops += self.dim * H * W
+        # W-MSA/SW-MSA
+        nW = H * W / self.window_size / self.window_size
+        flops += nW * self.attn.flops(self.window_size * self.window_size)
+        # mlp
+        flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio
+        # norm2
+        flops += self.dim * H * W
+        return flops
+
+
+class PatchMerging(nn.Module):
+    r""" Patch Merging Layer.
+    Args:
+        input_resolution (tuple[int]): Resolution of input feature.
+        dim (int): Number of input channels.
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+
+    def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.dim = dim
+        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)
+        self.norm = norm_layer(2 * dim)
+
+    def forward(self, x):
+        """
+        x: B, H*W, C
+        """
+        H, W = self.input_resolution
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+        assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even."
+
+        x = x.view(B, H, W, C)
+
+        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C
+        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C
+        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C
+        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C
+        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C
+        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C
+
+        x = self.reduction(x)
+        x = self.norm(x)
+
+        return x
+
+    def extra_repr(self) -> str:
+        return f"input_resolution={self.input_resolution}, dim={self.dim}"
+
+    def flops(self):
+        H, W = self.input_resolution
+        flops = (H // 2) * (W // 2) * 4 * self.dim * 2 * self.dim
+        flops += H * W * self.dim // 2
+        return flops
+
+
+class BasicLayer(nn.Module):
+    """ A basic Swin Transformer layer for one stage.
+    Args:
+        dim (int): Number of input channels.
+        input_resolution (tuple[int]): Input resolution.
+        depth (int): Number of blocks.
+        num_heads (int): Number of attention heads.
+        window_size (int): Local window size.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
+        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
+        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+        pretrained_window_size (int): Local window size in pre-training.
+    """
+
+    def __init__(self, dim, input_resolution, depth, num_heads, window_size,
+                 mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0.,
+                 drop_path=0., norm_layer=nn.LayerNorm, downsample=None, use_checkpoint=False,
+                 pretrained_window_size=0):
+
+        super().__init__()
+        self.dim = dim
+        self.input_resolution = input_resolution
+        self.depth = depth
+        self.use_checkpoint = use_checkpoint
+
+        # build blocks
+        self.blocks = nn.ModuleList([
+            SwinTransformerBlock(dim=dim, input_resolution=input_resolution,
+                                 num_heads=num_heads, window_size=window_size,
+                                 shift_size=0 if (i % 2 == 0) else window_size // 2,
+                                 mlp_ratio=mlp_ratio,
+                                 qkv_bias=qkv_bias,
+                                 drop=drop, attn_drop=attn_drop,
+                                 drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
+                                 norm_layer=norm_layer,
+                                 pretrained_window_size=pretrained_window_size)
+            for i in range(depth)])
+
+        # patch merging layer
+        if downsample is not None:
+            self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer)
+        else:
+            self.downsample = None
+
+    def forward(self, x, checkpoint_activations=False):
+        for blk in self.blocks:
+            if self.use_checkpoint or checkpoint_activations:
+                x = checkpoint(blk, x)
+            else:
+                x = blk(x)
+        if self.downsample is not None:
+            x = self.downsample(x)
+        return x
+
+    def extra_repr(self) -> str:
+        return f"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}"
+
+    def flops(self):
+        flops = 0
+        for blk in self.blocks:
+            flops += blk.flops()
+        if self.downsample is not None:
+            flops += self.downsample.flops()
+        return flops
+
+    def _init_respostnorm(self):
+        for blk in self.blocks:
+            nn.init.constant_(blk.norm1.bias, 0)
+            nn.init.constant_(blk.norm1.weight, 0)
+            nn.init.constant_(blk.norm2.bias, 0)
+            nn.init.constant_(blk.norm2.weight, 0)
+
+
+class PatchEmbed(nn.Module):
+    r""" Image to Patch Embedding
+    Args:
+        img_size (int): Image size.  Default: 224.
+        patch_size (int): Patch token size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        norm_layer (nn.Module, optional): Normalization layer. Default: None
+    """
+
+    def __init__(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):
+        super().__init__()
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        patches_resolution = [img_size[0] // patch_size[0], img_size[1] // patch_size[1]]
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.patches_resolution = patches_resolution
+        self.num_patches = patches_resolution[0] * patches_resolution[1]
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+        if norm_layer is not None:
+            self.norm = norm_layer(embed_dim)
+        else:
+            self.norm = None
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        # FIXME look at relaxing size constraints
+        assert H == self.img_size[0] and W == self.img_size[1], \
+            f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
+        x = self.proj(x).flatten(2).transpose(1, 2)  # B Ph*Pw C
+        if self.norm is not None:
+            x = self.norm(x)
+        return x
+
+    def flops(self):
+        Ho, Wo = self.patches_resolution
+        flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
+        if self.norm is not None:
+            flops += Ho * Wo * self.embed_dim
+        return flops
+
+class SwinTransformerConfig:
+    def __init__(self, img_size=224,
+                 patch_size=4,
+                 in_chans=3,
+                 embed_dim=96,
+                 depths=[2, 2, 6, 2],
+                 num_heads=[3, 6, 12, 24],
+                 window_size=7,
+                 mlp_ratio=4.,
+                 qkv_bias=True,
+                 drop_rate=0.,
+                 attn_drop_rate=0.,
+                 drop_path_rate=0.1,
+                 ape=False,
+                 patch_norm=True,
+                 pretrained_window_sizes=[0, 0, 0, 0],
+                 checkpoint_activations=False,
+                 num_classes=1000):
+        self.num_classes = num_classes
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+        self.depths = depths
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.mlp_ratio = mlp_ratio
+        self.qkv_bias = qkv_bias
+        self.drop_rate = drop_rate
+        self.attn_drop_rate = attn_drop_rate
+        self.drop_path_rate = drop_path_rate
+        self.norm_layer = nn.LayerNorm
+        self.ape = ape
+        self.patch_norm = patch_norm
+        self.checkpoint_activations = checkpoint_activations
+        self.pretrained_window_sizes = pretrained_window_sizes
+
+
+class SwinTransformerV2(BaseModel):
+    r""" Swin Transformer
+        A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -
+          https://arxiv.org/pdf/2103.14030
+    Args:
+        img_size (int | tuple(int)): Input image size. Default 224
+        patch_size (int | tuple(int)): Patch size. Default: 4
+        in_chans (int): Number of input image channels. Default: 3
+        num_classes (int): Number of classes for classification head. Default: 1000
+        embed_dim (int): Patch embedding dimension. Default: 96
+        depths (tuple(int)): Depth of each Swin Transformer layer.
+        num_heads (tuple(int)): Number of attention heads in different layers.
+        window_size (int): Window size. Default: 7
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4
+        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
+        drop_rate (float): Dropout rate. Default: 0
+        attn_drop_rate (float): Attention dropout rate. Default: 0
+        drop_path_rate (float): Stochastic depth rate. Default: 0.1
+        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
+        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False
+        patch_norm (bool): If True, add normalization after patch embedding. Default: True
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False
+        pretrained_window_sizes (tuple(int)): Pretrained window sizes of each layer.
+    """
+
+    def __init__(self, config, num_classes=1000,  **kwargs):
+        super().__init__(config, **kwargs)
+        swin_config = SwinTransformerConfig(**config)
+
+        embed_dim = swin_config.embed_dim
+        img_size = swin_config.img_size
+        drop_rate = swin_config.drop_rate
+        depths = swin_config.depths
+
+        self.num_classes = num_classes
+        self.num_layers = len(depths)
+        self.embed_dim = embed_dim
+        self.ape = swin_config.ape
+
+        self.patch_norm = swin_config.patch_norm
+        self.num_features = int(embed_dim * 2 ** (self.num_layers - 1))
+        self.mlp_ratio = swin_config.mlp_ratio
+
+        # split image into non-overlapping patches
+        self.patch_embed = PatchEmbed(
+            img_size=img_size, patch_size=swin_config.patch_size, in_chans=swin_config.in_chans, embed_dim=embed_dim,
+            norm_layer=swin_config.norm_layer if self.patch_norm else None)
+        num_patches = self.patch_embed.num_patches
+        patches_resolution = self.patch_embed.patches_resolution
+        self.patches_resolution = patches_resolution
+
+        # absolute position embedding
+        if self.ape:
+            self.absolute_pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dim))
+            trunc_normal_(self.absolute_pos_embed, std=.02)
+
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        # stochastic depth
+        dpr = [x.item() for x in torch.linspace(0, swin_config.drop_path_rate, sum(depths))]  # stochastic depth decay rule
+
+        # build layers
+        self.layers = nn.ModuleList()
+        for i_layer in range(self.num_layers):
+            layer = BasicLayer(dim=int(embed_dim * 2 ** i_layer),
+                               input_resolution=(patches_resolution[0] // (2 ** i_layer),
+                                                 patches_resolution[1] // (2 ** i_layer)),
+                               depth=depths[i_layer],
+                               num_heads=swin_config.num_heads[i_layer],
+                               window_size=swin_config.window_size,
+                               mlp_ratio=self.mlp_ratio,
+                               qkv_bias=swin_config.qkv_bias,
+                               drop=drop_rate, attn_drop=swin_config.attn_drop_rate,
+                               drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
+                               norm_layer=swin_config.norm_layer,
+                               downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
+                               use_checkpoint=swin_config.checkpoint_activations,
+                               pretrained_window_size=swin_config.pretrained_window_sizes[i_layer])
+            self.layers.append(layer)
+
+        self.norm = swin_config.norm_layer(self.num_features)
+        self.avgpool = nn.AdaptiveAvgPool1d(1)
+        self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity()
+
+        self.apply(self._init_weights)
+        for bly in self.layers:
+            bly._init_respostnorm()
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {'absolute_pos_embed'}
+
+    @torch.jit.ignore
+    def no_weight_decay_keywords(self):
+        return {"cpb_mlp", "logit_scale", 'relative_position_bias_table'}
+
+    def forward_features(self, x):
+        x = self.patch_embed(x)
+        if self.ape:
+            x = x + self.absolute_pos_embed
+        x = self.pos_drop(x)
+
+        for layer in self.layers:
+
+            x = layer(x, checkpoint_activations=self.config["checkpoint_activations"])
+
+        x = self.norm(x)  # B L C
+        x = self.avgpool(x.transpose(1, 2))  # B C 1
+        x = torch.flatten(x, 1)
+        return x
+
+    def compute_loss(self, pred, labels):
+        loss_func = nn.CrossEntropyLoss()
+        loss = loss_func(pred, labels)
+        return loss
+
+    def forward(self, images=None, labels=None, **kwargs):
+        x = self.forward_features(images)
+        x = self.head(x)
+        return_data = {"logits": x}
+        if labels is not None:
+            loss = self.compute_loss(x, labels)
+            return_data["loss"] = loss
+        return return_data
+
+    def flops(self):
+        flops = 0
+        flops += self.patch_embed.flops()
+        for i, layer in enumerate(self.layers):
+            flops += layer.flops()
+        flops += self.num_features * self.patches_resolution[0] * self.patches_resolution[1] // (2 ** self.num_layers)
+        flops += self.num_features * self.num_classes
+        return flops
+
+    def load_weights(self, checkpoint_path):
+        self.load_state_dict(torch.load(checkpoint_path, map_location="cpu"))
diff --git a/flagai/mp_tools.py b/flagai/mp_tools.py
index de5394af..b4e9a108 100644
--- a/flagai/mp_tools.py
+++ b/flagai/mp_tools.py
@@ -305,6 +305,111 @@ def change_pytorch_model_mp_from_1_to_n_new(model_name_brief, checkpoint: str, t
                 torch.save(d_new, filename)
 
 
+def change_pytorch_model_mp_from_n_to_1(model_name_brief, checkpoint):
+    #model merge
+    trans_keys = from_1_to_n_models.get(model_name_brief, None)
+    if trans_keys is None:
+        print(f"Not support the model_name: {model_name_brief}")
+        os._exit(0)
+
+    assert os.path.isdir(checkpoint)
+    filenames = os.listdir(checkpoint)
+    filenames = [
+        filename for filename in filenames
+        if filename.startswith("pytorch_model")
+    ]
+
+    filenames = sorted(filenames)
+    if 'pytorch_model.bin' in filenames:
+        print("no need to merge")
+        exit(0)
+
+    filenames = [os.path.join(checkpoint, x) for x in filenames]
+
+    if checkpoint[-1] == '/':
+        new_checkpoint = checkpoint[:-1]
+    else:
+        new_checkpoint = checkpoint
+
+    preserve_keys = [
+        "lr_scheduler",
+        "skipped_steps",
+        "global_steps",
+        "global_samples",
+        "dp_world_size",
+        "iteration",
+        "client_lr_scheduler",
+        "np_rng_state",
+        "random_rng_state",
+        "torch_rng_state",
+        "cuda_rng_state",
+        "rng_tracker_states",
+    ]
+
+    print("Decrease MP size.")
+    ratio = len(filenames)
+    for i in range(1):
+        start = ratio * i
+        end = ratio * (i + 1)
+        d = torch.load(filenames[start], map_location='cpu')
+
+        if "module" in d:
+            d = d["module"]
+
+        for j in range(start+1, end):
+            d_new = torch.load(filenames[j], map_location='cpu')
+            if "module" in d_new:
+                d_new = d_new["module"]
+
+                for k, v in d_new.items():
+                    assert len(v.shape) < 3
+                    for keys in trans_keys:
+                        if keys in k:
+                            # find a key to concat
+                            dim = trans_keys[keys]
+                            if len(v.shape) == 2:
+                                if dim == 30:
+                                    size_1 = d[k].shape[0] // 3
+                                    size_2 = v.shape[0] // 3
+                                    target = d[k]
+                                    d[k] = torch.cat([
+                                        target[:size_1, :], v[:size_2, :],
+                                        target[size_1:size_1 * 2, :],
+                                        v[size_2:size_2 * 2, :], target[size_1 * 2:, :],
+                                        v[size_2 * 2:, :]
+                                    ], 0)
+                                    break
+
+                                elif dim == 0:
+                                    d[k] = torch.cat([d[k], v], 0)
+                                    break
+
+                                elif dim == 1:
+                                    d[k] = torch.cat([d[k], v], 1)
+                                    break
+
+                            elif len(v.shape) == 1:
+                                if dim == 30:
+                                    size_1 = d[k].shape[0] // 3
+                                    size_2 = v.shape[0] // 3
+                                    target = d[k]
+                                    d[k] = torch.cat([
+                                        target[:size_1], v[:size_2], target[size_1:size_1 * 2],
+                                        v[size_2:size_2 * 2], target[size_1 * 2:],
+                                        v[size_2 * 2:]
+                                    ], 0)
+
+                                    break
+
+                                else :
+                                    d[k] = torch.cat([d[k], v], 0)
+                                    break
+
+        filename = os.path.join(new_checkpoint,
+                                "pytorch_model.bin")
+        print(f"merge succeed: {filename}")
+        torch.save(d, filename)
+
 if __name__ == "__main__":
     change_pytorch_model_mp_from_1_to_n(
         '/mnt/test_10b_models/state_dict/GLM-10b-en', 2)
diff --git a/flagai/trainer.py b/flagai/trainer.py
index f2fd8dd1..4a7dbef9 100644
--- a/flagai/trainer.py
+++ b/flagai/trainer.py
@@ -481,7 +481,6 @@ def train(self,
         best_score = float('inf')
         if len(self.metric_methods) > 0:
             best_score = -best_score
-
         for epoch in range(self.epochs):
             # log_dist('working on epoch {} ...'.format(epoch), [0])
             # Set the data loader epoch to shuffle the index iterator.
@@ -961,7 +960,7 @@ def evaluate(self,
                 all_losses = self._gather_all_mpu(all_losses)
 
             if all_losses.device != torch.device('cpu'):
-                all_losses = all_losses.cpu().detach().numpy()[0]
+                all_losses = all_losses.mean().cpu().detach().numpy()
 
             for i in range(len(self.metric_methods)):
                 eval_method = self.metric_methods[i][1]
diff --git a/requirements.txt b/requirements.txt
index 62c18e1f..4b2dbdf5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,3 +9,4 @@ transformers == 4.20.1
 datasets == 2.0.0
 setuptools == 59.5.0
 protobuf == 3.20.1
+ftfy == 6.1.1
diff --git a/setup.py b/setup.py
index f1055c2c..5df713be 100644
--- a/setup.py
+++ b/setup.py
@@ -5,8 +5,8 @@
 
 setup(
     name="flagai",
-    version="v1.2.0",
-    description="FlagAI aims to help researchers and developers to freely train and test large-scale models for NLP tasks.",
+    version="v1.3.0",
+    description="FlagAI aims to help researchers and developers to freely train and test large-scale models for NLP/CV/VL tasks.",
     long_description=open("README.md", encoding="utf-8").read(),
     long_description_content_type="text/markdown",
     author="FlagAI-Open",


Test patch:
diff --git a/flagai/test_utils.py b/flagai/test_utils.py
index a0aed406..83dacde3 100644
--- a/flagai/test_utils.py
+++ b/flagai/test_utils.py
@@ -14,10 +14,10 @@ def build_input_from_ids(text_a_ids=None,
                          mask_id=None,
                          masked_lm=False):
     if mask_id is None:
-        mask_id = tokenizer.get_command('MASK').Id
-    eos_id = tokenizer.get_command('eos').Id
-    cls_id = tokenizer.get_command('ENC').Id
-    sep_id = tokenizer.get_command('sep').Id
+        mask_id = tokenizer.get_command_id('MASK')
+    eos_id = tokenizer.get_command_id('eos')
+    cls_id = tokenizer.get_command_id('cls')
+    sep_id = tokenizer.get_command_id('sep')
     ids = []
     types = []
     paddings = []
@@ -61,7 +61,7 @@ def build_input_from_ids(text_a_ids=None,
     block_position_ids = [0] * len(ids)
     # Piece
     if add_piece or answer_ids is not None:
-        sop_id = tokenizer.get_command('sop').Id
+        sop_id = tokenizer.get_command_id('sop')
         mask_position = ids.index(
             mask_id
         ) if not args.sentinel_token else args.max_position_embeddings
diff --git a/tests/bak_test_glm_superglue.py b/tests/bak_test_glm_superglue.py
deleted file mode 100644
index 8cea1c21..00000000
--- a/tests/bak_test_glm_superglue.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# Copyright © 2022 BAAI. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License")
-import torch
-from flagai.trainer import Trainer
-from flagai.model.glm_model import GLMForSingleTokenCloze, GLMForMultiTokenCloze
-from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer
-from flagai.data.dataset import SuperGlueDataset
-from flagai.test_utils import CollateArguments
-from flagai.data.dataset.superglue.control import DEFAULT_METRICS, MULTI_TOKEN_TASKS, CH_TASKS
-import unittest
-from flagai.data.dataset import ConstructSuperglueStrategy
-
-
-class TrainerTestCase(unittest.TestCase):
-
-    def test_init_trainer_pytorch(self):
-        for task_name in [
-                'boolq', 'cb', 'copa', 'multirc', 'rte', 'wic', 'wsc', 'afqmc',
-                'tnews'
-        ]:
-            trainer = Trainer(env_type='pytorch',
-                              epochs=1,
-                              batch_size=1,
-                              eval_interval=100,
-                              log_interval=50,
-                              experiment_name='glm_large',
-                              pytorch_device='cuda',
-                              load_dir=None,
-                              fp16=True,
-                              lr=1e-4,
-                              save_interval=10)
-            print("downloading...")
-
-            cl_args = CollateArguments()
-            cl_args.multi_token = task_name in MULTI_TOKEN_TASKS
-            if task_name in CH_TASKS:
-                model_name = 'GLM-large-ch'
-                #lm_model = GLMModel.from_pretrain(model_name='GLM-large-ch')
-                tokenizer = GLMLargeChTokenizer()
-                # tokenizer = GLMBertWordPieceTokenizer(tokenizer_model_type='BERT-base-ch')
-            else:
-                model_name = 'GLM-large-en'
-                #lm_model = GLMModel.from_pretrain(model_name='GLM-large-en')
-                tokenizer = GLMLargeEnWordPieceTokenizer()
-
-            if cl_args.multi_token:
-                model = GLMForMultiTokenCloze.from_pretrain(
-                    model_name=model_name, only_download_config=True)
-            else:
-                model = GLMForSingleTokenCloze.from_pretrain(
-                    model_name=model_name, only_download_config=True)
-
-            train_dataset = SuperGlueDataset(task_name=task_name,
-                                             data_dir='./datasets/',
-                                             dataset_type='train',
-                                             tokenizer=tokenizer)
-            train_dataset.example_list = train_dataset.example_list[:1]
-            collate_fn = ConstructSuperglueStrategy(cl_args,
-                                                    tokenizer,
-                                                    task_name=task_name)
-
-            valid_dataset = SuperGlueDataset(task_name=task_name,
-                                             data_dir='./datasets/',
-                                             dataset_type='dev',
-                                             tokenizer=tokenizer)
-            valid_dataset.example_list = valid_dataset.example_list[:1]
-            print(task_name)
-            metric_methods = DEFAULT_METRICS[task_name]
-            trainer.train(model,
-                          collate_fn=collate_fn,
-                          train_dataset=train_dataset,
-                          valid_dataset=valid_dataset,
-                          metric_methods=metric_methods)
-
-
-def suite():
-    suite = unittest.TestSuite()
-    suite.addTest(TrainerTestCase('test_init_trainer_pytorch'))
-    return suite
-
-
-if __name__ == '__main__':
-    runner = unittest.TextTestRunner()
-    runner.run(suite())
diff --git a/tests/bak_test_superglue.py b/tests/bak_test_superglue.py
index 306abb38..ebb3a9ad 100644
--- a/tests/bak_test_superglue.py
+++ b/tests/bak_test_superglue.py
@@ -4,7 +4,7 @@
 import torch
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze, GLMForMultiTokenCloze, GLMForSequenceClassification
-from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer, BertWordPieceTokenizer, T5BPETokenizer, ROBERTATokenizer, OPTTokenizer, CPMTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
 from flagai.data.dataset.superglue.control import DEFAULT_METRICS, MULTI_TOKEN_TASKS, CH_TASKS
@@ -15,12 +15,9 @@
 class TrainerTestCase(unittest.TestCase):
 
     def test_init_trainer_pytorch(self):
-        # for task_name in [
-        #         'boolq', 'cb', 'copa', 'multirc', 'rte', 'wic', 'wsc', 'afqmc',
-        #         'tnews', 'qqp', 'cola', 'mnli', 'qnli'
-        # ]:
         for task_name in [
-            'boolq'
+                'boolq', 'cb', 'copa', 'multirc', 'rte', 'wic', 'wsc', 'afqmc',
+                'tnews', 'qqp', 'cola', 'mnli', 'qnli'
         ]:
             trainer = Trainer(env_type='pytorch',
                               epochs=1,
@@ -39,17 +36,10 @@ def test_init_trainer_pytorch(self):
             cl_args.multi_token = task_name in MULTI_TOKEN_TASKS
             if task_name in CH_TASKS:
                 model_name = 'GLM-large-ch'
-                tokenizer = GLMLargeChTokenizer()
             else:
                 model_name = 'GLM-large-en'
-                # tokenizer = GLMLargeEnWordPieceTokenizer()
-                # tokenizer = BertWordPieceTokenizer()
-                tokenizer = T5BPETokenizer()
-                # tokenizer = ROBERTATokenizer()
-                # tokenizer = OPTTokenizer()
-                # tokenizer = CPMTokenizer()
-
 
+            tokenizer = Tokenizer.from_pretrained(model_name)
             if cl_args.cloze_eval:
                 if cl_args.multi_token:
                     model = GLMForMultiTokenCloze.from_pretrain(
@@ -65,24 +55,10 @@ def test_init_trainer_pytorch(self):
                                              data_dir='./datasets/',
                                              dataset_type='train',
                                              tokenizer=tokenizer)
-            # print(train_dataset[0])
+            train_dataset.example_list = train_dataset.example_list[:1]
             collate_fn = ConstructSuperglueStrategy(cl_args,
                                                     tokenizer,
                                                     task_name=task_name)
-            # import torch
-            # loader = torch.utils.data.DataLoader(train_dataset,
-            #                                      batch_size=1,
-            #                                      shuffle=False,
-            #                                      num_workers=1,
-            #                                      drop_last=False,
-            #                                      pin_memory=False,
-            #                                      collate_fn=collate_fn)
-            # for data_iterator in loader:
-            #     for key, value in data_iterator.items():
-            #         print(key, value)
-            #     break
-            train_dataset.example_list = train_dataset.example_list[:1]
-
 
             valid_dataset = SuperGlueDataset(task_name=task_name,
                                              data_dir='./datasets/',
@@ -106,4 +82,4 @@ def suite():
 
 if __name__ == '__main__':
     runner = unittest.TextTestRunner()
-    runner.run(suite())
+    runner.run(suite())
\ No newline at end of file
diff --git a/tests/test_bert.py b/tests/test_bert.py
index d85a08e0..d5c0d0c2 100644
--- a/tests/test_bert.py
+++ b/tests/test_bert.py
@@ -4,6 +4,7 @@
 from flagai.auto_model.auto_loader import AutoLoader
 from flagai.model.predictor.predictor import Predictor
 import torch
+from flagai.data.tokenizer import Tokenizer
 from flagai.model.bert_model import BertModel, BertForSeq2seq, \
                                     BertForSequenceLabeling, \
                                     BertForSequenceLabelingGP, \
@@ -23,8 +24,8 @@ def setUp(self) -> None:
                        BertForSequenceLabelingCRF]
         self.model_name = "RoBERTa-base-ch"
         self.bert_path = "./checkpoints/RoBERTa-base-ch/config.json"
-        self.tokenizer = BertTokenizer("./checkpoints/RoBERTa-base-ch/vocab.txt")
-
+        # self.tokenizer = BertTokenizer("./checkpoints/RoBERTa-base-ch/vocab.txt")
+        self.tokenizer = Tokenizer.from_pretrained(self.model_name)
         print("loading bert model successfully!")
 
     def test_model_predict(self):
diff --git a/tests/test_glm_large_ch.py b/tests/test_glm_large_ch.py
index d75573c1..55552425 100644
--- a/tests/test_glm_large_ch.py
+++ b/tests/test_glm_large_ch.py
@@ -4,7 +4,7 @@
 from flagai.model.predictor.predictor import Predictor
 import torch
 from flagai.model.glm_model import GLMForSeq2Seq
-from flagai.data.tokenizer.glm_large_ch.glm_large_ch_tokenizer import GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 import unittest
 
 class GLMLargeChTestCase(unittest.TestCase):
@@ -12,7 +12,7 @@ class GLMLargeChTestCase(unittest.TestCase):
     def setUp(self) -> None:
 
         self.model = GLMForSeq2Seq.init_from_json("./checkpoints/GLM-large-ch/config.json")
-        self.tokenizer = GLMLargeChTokenizer("./checkpoints/GLM-large-ch/cog-pretrain.model")
+        self.tokenizer = Tokenizer.from_pretrained("GLM-large-ch")
         print("loading bert model successfully!")
 
     def test_model_predict(self):
diff --git a/tests/test_glm_seq2seq.py b/tests/test_glm_seq2seq.py
index 0d008304..e12fa1fe 100644
--- a/tests/test_glm_seq2seq.py
+++ b/tests/test_glm_seq2seq.py
@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSeq2Seq
-from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.data.dataset import Seq2SeqDataset
 from flagai.data.dataset.superglue.control import DEFAULT_METRICS, CH_TASKS
 from flagai.data.dataset import ConstructSeq2seqStrategy
@@ -30,18 +30,17 @@ def test_init_trainer_pytorch(self):
         print("downloading...")
 
         if task_name in CH_TASKS:
-            tokenizer = GLMLargeChTokenizer()
             model_name = 'GLM-large-ch'
         else:
-            tokenizer = GLMLargeEnWordPieceTokenizer()
             model_name = 'GLM-large-en'
+        tokenizer = Tokenizer.from_pretrained(model_name)
 
         train_dataset = Seq2SeqDataset(task_name=task_name,
-                                       data_dir='./datasets/',
+                                       data_dir='./data/',
                                        dataset_type='train',
                                        tokenizer=tokenizer)
         valid_dataset = Seq2SeqDataset(task_name=task_name,
-                                       data_dir='./datasets/',
+                                       data_dir='./data/',
                                        dataset_type='dev',
                                        tokenizer=tokenizer)
 
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
index 4832c700..c1bc77c2 100644
--- a/tests/test_tokenizer.py
+++ b/tests/test_tokenizer.py
@@ -2,20 +2,13 @@
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
 import unittest
-from flagai.data.tokenizer import GLMLargeChTokenizer
-from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer
-from flagai.data.tokenizer import GLM10bENBPETokenizer
-from flagai.data.tokenizer import T5BPETokenizer
-from flagai.data.tokenizer import ROBERTATokenizer
-from flagai.data.tokenizer import BertWordPieceTokenizer
-from flagai.data.tokenizer import OPTTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.auto_model.auto_loader import AutoLoader
 
 class TokenizerTestCase(unittest.TestCase):
 
     def test_tokenizer_glm_large_ch(self):
-        tokenizer = GLMLargeChTokenizer()
-
+        tokenizer = Tokenizer.from_pretrained("GLM-large-ch")
         self.assertEqual(tokenizer.TokenToId("人"), 43371, 'Token id "人" error')
         self.assertEqual(tokenizer.EncodeAsIds("今天吃饭吃了肯德基"),
                          [3378, 1567, 2613, 20282], 'EncodeAsIds Error')
@@ -23,41 +16,40 @@ def test_tokenizer_glm_large_ch(self):
                          '今天吃饭吃了肯德基', 'DecodeIds Error')
 
     def test_tokenizer_GLM_large_en(self):
-        tokenizer = GLMLargeEnWordPieceTokenizer()
-        print(tokenizer.EncodeAsIds("today is a nice day and"))
+        tokenizer = Tokenizer.from_pretrained("GLM-large-en")
         self.assertEqual(tokenizer.TokenToId("day"), 2154, '')
         self.assertEqual(tokenizer.EncodeAsIds("fried chicken makes me happy"),
                          [13017, 7975, 3084, 2033, 3407], '')
         self.assertEqual(tokenizer.DecodeIds([13017, 7975, 3084, 2033, 3407]),
                          'fried chicken makes me happy', 'DecodeIds Error')
 
-    def test_tokenizer_glm_10b_en(self):
-        tokenizer = GLM10bENBPETokenizer()
-        self.assertEqual(tokenizer.TokenToId("day"), 820, '')
-        self.assertEqual(tokenizer.EncodeAsIds("fried chicken makes me happy"),
-                         [25520, 9015, 1838, 502, 3772], '')
-        self.assertEqual(tokenizer.DecodeIds([25520, 9015, 1838, 502, 3772]),
-                         'fried chicken makes me happy', 'DecodeIds Error')
-
+    # def test_tokenizer_glm_10b_en(self):
+    #     tokenizer = Tokenizer.from_pretrained("GLM-10b-en")
+    #     self.assertEqual(tokenizer.TokenToId("day"), 820, '')
+    #     self.assertEqual(tokenizer.EncodeAsIds("fried chicken makes me happy"),
+    #                      [25520, 9015, 1838, 502, 3772], '')
+    #     self.assertEqual(tokenizer.DecodeIds([25520, 9015, 1838, 502, 3772]),
+    #                      'fried chicken makes me happy', 'DecodeIds Error')
+    
     def test_tokenizer_t5(self):
-        tokenizer = T5BPETokenizer(tokenizer_model_type='t5-base')
+        tokenizer = Tokenizer.from_pretrained('t5-base-en')
         self.assertEqual(tokenizer.TokenToId("day"), 1135, '')
         self.assertEqual(tokenizer.EncodeAsIds("fried chicken makes me happy"),
                          [3, 7704, 3832, 656, 140, 1095], '')
         self.assertEqual(tokenizer.DecodeIds([3, 7704, 3832, 656, 140, 1095]),
                          'fried chicken makes me happy', 'DecodeIds Error')
-
-    def test_tokenizer_roberta(self):
-        tokenizer = ROBERTATokenizer(tokenizer_model_type='roberta-base')
-        self.assertEqual(tokenizer.TokenToId("day"), 1208, '')
-        self.assertEqual(tokenizer.EncodeAsIds("fried chicken makes me happy"),
-                         [21209, 5884, 817, 162, 1372], '')
-        self.assertEqual(tokenizer.DecodeIds([21209, 5884, 817, 162, 1372]),
-                         'fried chicken makes me happy', 'DecodeIds Error')
+    # # #
+    # def test_tokenizer_roberta(self):
+    #     tokenizer = ROBERTATokenizer(tokenizer_model_type='roberta-base')
+    #     tokenizer = Tokenizer.from_pretrained('t5-base-en')
+    #     self.assertEqual(tokenizer.TokenToId("day"), 1208, '')
+    #     self.assertEqual(tokenizer.EncodeAsIds("fried chicken makes me happy"),
+    #                      [21209, 5884, 817, 162, 1372], '')
+    #     self.assertEqual(tokenizer.DecodeIds([21209, 5884, 817, 162, 1372]),
+    #                      'fried chicken makes me happy', 'DecodeIds Error')
 
     def test_tokenizer_bert(self):
-        tokenizer = BertWordPieceTokenizer(
-            tokenizer_model_type='bert-large-uncased')
+        tokenizer = Tokenizer.from_pretrained('BERT-base-en')
         self.assertEqual(tokenizer.TokenToId("day"), 2154, '')
         self.assertEqual(tokenizer.EncodeAsIds("fried chicken makes me happy"),
                          [13017, 7975, 3084, 2033, 3407], '')
@@ -72,15 +64,15 @@ def test_tokenizer_cpm1(self):
         tokenizer = loader.get_tokenizer()
         self.assertEqual(tokenizer.encode("day"), [8, 8275], '')
         self.assertEqual(tokenizer.encode("fried chicken makes me happy"),
-                         [2487, 27385, 8, 10, 9291, 9412, 3531, 8, 10, 14588, 289, 8, 10, 4406, 8, 10, 25239], '')
-        self.assertEqual(tokenizer.decode([2487, 27385, 8, 10, 9291, 9412, 3531, 8, 10, 14588, 289, 8, 10, 4406, 8, 10, 25239]),
+                         [2487, 27385, 9291, 9412, 3531, 14588, 289, 4406, 25239], '')
+        self.assertEqual(tokenizer.decode([2487, 27385, 9291, 9412, 3531, 14588, 289, 4406, 25239]),
                          'fried chicken makes me happy', 'DecodeIds Error')
 
     def test_tokenizer_opt(self):
-        tokenizer = OPTTokenizer(tokenizer_model_type="facebook/opt-125m")
-        self.assertEqual(tokenizer.get_vocab()["day"], 1208, '')
+        tokenizer = Tokenizer.from_pretrained('opt-125m-en')
+        self.assertEqual(tokenizer.encode("day"), [1208], '')
         self.assertEqual(tokenizer.encode_plus("fried chicken makes me happy")["input_ids"],
-                         [2, 21209, 5884, 817, 162, 1372], '')
+                         [21209, 5884, 817, 162, 1372], '')
         self.assertEqual(tokenizer.decode([21209, 5884, 817, 162, 1372]),
                          'fried chicken makes me happy', 'DecodeIds Error')
 
@@ -89,9 +81,9 @@ def suite():
     suite = unittest.TestSuite()
     suite.addTest(TokenizerTestCase('test_tokenizer_GLM_large_ch'))
     suite.addTest(TokenizerTestCase('test_tokenizer_GLM_large_en'))
-    suite.addTest(TokenizerTestCase('test_tokenizer_glm_10_en'))
+    # suite.addTest(TokenizerTestCase('test_tokenizer_glm_10_en'))
     suite.addTest(TokenizerTestCase('test_tokenizer_t5'))
-    suite.addTest(TokenizerTestCase('test_tokenizer_roberta'))
+    # suite.addTest(TokenizerTestCase('test_tokenizer_roberta'))
     suite.addTest(TokenizerTestCase('test_tokenizer_bert'))
     suite.addTest(TokenizerTestCase('test_tokenizer_cpm1'))
     suite.addTest(TokenizerTestCase('test_tokenizer_opt'))