Spaces:
Sleeping
Sleeping
Upload better_transformer.py
Browse files- better_transformer.py +1 -1
better_transformer.py
CHANGED
|
@@ -394,7 +394,7 @@ def generate(model, tokenizer, device, method=None, k=None,
|
|
| 394 |
res = re.sub(r'“', '"', res)
|
| 395 |
res = re.sub(r'’', "'", res)
|
| 396 |
res = re.sub(r'â€', '"', res)
|
| 397 |
-
res = res + "
|
| 398 |
return res
|
| 399 |
|
| 400 |
|
|
|
|
| 394 |
res = re.sub(r'“', '"', res)
|
| 395 |
res = re.sub(r'’', "'", res)
|
| 396 |
res = re.sub(r'â€', '"', res)
|
| 397 |
+
res = res + " <|endoftext|>" ## better end token
|
| 398 |
return res
|
| 399 |
|
| 400 |
|