coyotte508 HF staff commited on
Commit
189b280
1 Parent(s): d589b11

♻️ Simplify / improve token removal

Browse files
src/lib/components/chat/ChatMessage.svelte CHANGED
@@ -9,8 +9,7 @@
9
 
10
  function sanitizeMd(md: string) {
11
  return md
12
- .replaceAll(/^(<\|startoftext\|$|<\|startoftext$|<\|$|<$)/g, "")
13
- .replaceAll(/(<\|endoftext\|$|<\|endoftext$)/g, "")
14
  .replaceAll(/<\|[a-z]+\|>/g, " ")
15
  .trim()
16
  .replaceAll("&", "&amp;")
 
9
 
10
  function sanitizeMd(md: string) {
11
  return md
12
+ .replace(/<\|[a-z]*$/, "")
 
13
  .replaceAll(/<\|[a-z]+\|>/g, " ")
14
  .trim()
15
  .replaceAll("&", "&amp;")