Update modeling_internlm_xcomposer2.py
Browse files
modeling_internlm_xcomposer2.py
CHANGED
|
@@ -60,6 +60,7 @@ class InternLMXComposer2ForCausalLM(InternLM2PreTrainedModel):
|
|
| 60 |
self.tokenizer = None
|
| 61 |
|
| 62 |
self.eoh = '<TOKENS_UNUSED_0>' # end of human
|
|
|
|
| 63 |
|
| 64 |
self.max_length = config.max_length
|
| 65 |
print(f'Set max length to {self.max_length}')
|
|
|
|
| 60 |
self.tokenizer = None
|
| 61 |
|
| 62 |
self.eoh = '<TOKENS_UNUSED_0>' # end of human
|
| 63 |
+
self.eoa = '<TOKENS_UNUSED_145>' # end of assistant
|
| 64 |
|
| 65 |
self.max_length = config.max_length
|
| 66 |
print(f'Set max length to {self.max_length}')
|