myownskyW7 commited on
Commit
1b20edf
1 Parent(s): eea2f56

fix eoa and eoh

Browse files
Files changed (1) hide show
  1. modeling_InternLM_XComposer.py +2 -9
modeling_InternLM_XComposer.py CHANGED
@@ -101,15 +101,8 @@ conversation
101
 
102
  self.tokenizer = None
103
 
104
- @property
105
- def eoh(self):
106
- return self.tokenizer.decode(torch.Tensor([103027]),
107
- skip_special_tokens=True)
108
-
109
- @property
110
- def eoa(self):
111
- return self.tokenizer.decode(torch.Tensor([103028]),
112
- skip_special_tokens=True)
113
 
114
  def maybe_autocast(self, dtype=torch.float16):
115
  # if on cpu, don't use autocast
 
101
 
102
  self.tokenizer = None
103
 
104
+ self.eoh = '<TOKENS_UNUSED_0>' # end of human
105
+ self.eoa = '<TOKENS_UNUSED_1>' # end of assistant
 
 
 
 
 
 
 
106
 
107
  def maybe_autocast(self, dtype=torch.float16):
108
  # if on cpu, don't use autocast