Update modeling_internlm_xcomposer2.py

modeling_internlm_xcomposer2.py

@@ -57,8 +57,10 @@ class InternLMXComposer2ForCausalLM(InternLM2PreTrainedModel):
         self.vocab_size = config.vocab_size
         self.output = nn.Linear(
             config.hidden_size, config.vocab_size, bias=False)
-        # self.tokenizer = None
+        self.tokenizer = None
 
+        self.eoh = '<TOKENS_UNUSED_0>'  # end of human
+        
         self.max_length = config.max_length
         print(f'Set max length to {self.max_length}')
         # Initialize weights and apply final processing