Roaoch commited on
Commit
d32aebe
1 Parent(s): 549722f

Remove special tokens from responce

Browse files
Files changed (1) hide show
  1. src/cyberclaasic.py +1 -1
src/cyberclaasic.py CHANGED
@@ -55,5 +55,5 @@ class CyberClassic(torch.nn.Module):
55
  generation_config=self.generation_config,
56
  )
57
 
58
- decoded = self.tokenizer.batch_decode(output)
59
  return decoded[0][length:].strip()
 
55
  generation_config=self.generation_config,
56
  )
57
 
58
+ decoded = self.tokenizer.batch_decode(output, skip_special_tokens=True)
59
  return decoded[0][length:].strip()