qminh369 commited on
Commit
ba250a6
1 Parent(s): 339d283

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -15,7 +15,7 @@ compressors = {
15
 
16
  tokenizer = tiktoken.encoding_for_model("gpt-4")
17
 
18
- def compress(original_prompt, compression_rate, base_model="xlm-roberta", force_tokens=['\n'], chunk_end_tokens=['.', '\n']):
19
  if '\\n' in force_tokens:
20
  idx = force_tokens.index('\\n')
21
  force_tokens[idx] = '\n'
 
15
 
16
  tokenizer = tiktoken.encoding_for_model("gpt-4")
17
 
18
+ def compress(original_prompt, compression_rate, base_model="xlm-roberta", force_tokens = ['. ', ', '], chunk_end_tokens=['.', '\n']):
19
  if '\\n' in force_tokens:
20
  idx = force_tokens.index('\\n')
21
  force_tokens[idx] = '\n'