wjjessen commited on
Commit
c49d373
1 Parent(s): 0f19029

add first 500 tokens

Browse files
Files changed (1) hide show
  1. app.py +4 -0
app.py CHANGED
@@ -367,6 +367,10 @@ def main():
367
  first_10_tokens = input_ids[:10]
368
  first_10_tokens_text = tokenizer.convert_ids_to_tokens(first_10_tokens)
369
  st.write(first_10_tokens_text)
 
 
 
 
370
  st.write("[RecursiveCharacterTextSplitter](%s) parameters used:" % url)
371
  st.write(
372
  "        chunk_size=%s"
 
367
  first_10_tokens = input_ids[:10]
368
  first_10_tokens_text = tokenizer.convert_ids_to_tokens(first_10_tokens)
369
  st.write(first_10_tokens_text)
370
+ st.write("First 500 tokens:")
371
+ first_500_tokens = input_ids[:500]
372
+ first_500_tokens_text = tokenizer.convert_ids_to_tokens(first_500_tokens)
373
+ st.write(first_500_tokens_text)
374
  st.write("[RecursiveCharacterTextSplitter](%s) parameters used:" % url)
375
  st.write(
376
  "        chunk_size=%s"