Spaces:
Paused
Paused
Commit
·
189b280
1
Parent(s):
d589b11
♻️ Simplify / improve token removal
Browse files
src/lib/components/chat/ChatMessage.svelte
CHANGED
|
@@ -9,8 +9,7 @@
|
|
| 9 |
|
| 10 |
function sanitizeMd(md: string) {
|
| 11 |
return md
|
| 12 |
-
.
|
| 13 |
-
.replaceAll(/(<\|endoftext\|$|<\|endoftext$)/g, "")
|
| 14 |
.replaceAll(/<\|[a-z]+\|>/g, " ")
|
| 15 |
.trim()
|
| 16 |
.replaceAll("&", "&")
|
|
|
|
| 9 |
|
| 10 |
function sanitizeMd(md: string) {
|
| 11 |
return md
|
| 12 |
+
.replace(/<\|[a-z]*$/, "")
|
|
|
|
| 13 |
.replaceAll(/<\|[a-z]+\|>/g, " ")
|
| 14 |
.trim()
|
| 15 |
.replaceAll("&", "&")
|