Update modeling_internlm_xcomposer2.py

modeling_internlm_xcomposer2.py

@@ -100,12 +100,12 @@ class InternLMXComposer2ForCausalLM(InternLM2PreTrainedModel):
         return self.model
 
     def encode_text(self, tokenizer, text, add_special_tokens=False):
-        # token = self.tokenizer(
-        #     text, return_tensors='pt',
-        #     add_special_tokens=add_special_tokens).input_ids.to(self.device)
-        # embs = self.model.tok_embeddings(token)
-        # return embs
-        return type(tokenizer)
+        token = tokenizer(
+            text, return_tensors='pt',
+            add_special_tokens=add_special_tokens).input_ids.to(self.device)
+        embs = self.model.tok_embeddings(token)
+        return embs
+        # return type(tokenizer)
 
     def encode_img(self, image):
         if image is None: