coyotte508 HF staff commited on
Commit
a11cd21
1 Parent(s): 97b65ef

🎨 Remove more stopwords (#102)

Browse files
src/lib/components/chat/ChatMessage.svelte CHANGED
@@ -9,8 +9,10 @@
9
 
10
  function sanitizeMd(md: string) {
11
  return md
12
- .replaceAll(/^(<\|startoftext\|>|<\|startoftext\|$|<\|startoftext$|<\|$|<$)/g, "")
13
- .replaceAll(/(<\|endoftext\|>|<\|endoftext\|$|<\|endoftext$)/g, "")
 
 
14
  .replaceAll("&", "&amp;")
15
  .replaceAll("<", "&lt;");
16
  }
 
9
 
10
  function sanitizeMd(md: string) {
11
  return md
12
+ .replaceAll(/^(<\|startoftext\|$|<\|startoftext$|<\|$|<$)/g, "")
13
+ .replaceAll(/(<\|endoftext\|$|<\|endoftext$)/g, "")
14
+ .replaceAll(/<\|[a-z]+\|>/g, " ")
15
+ .trim()
16
  .replaceAll("&", "&amp;")
17
  .replaceAll("<", "&lt;");
18
  }