add first 500 tokens
Browse files
app.py
CHANGED
@@ -367,6 +367,10 @@ def main():
|
|
367 |
first_10_tokens = input_ids[:10]
|
368 |
first_10_tokens_text = tokenizer.convert_ids_to_tokens(first_10_tokens)
|
369 |
st.write(first_10_tokens_text)
|
|
|
|
|
|
|
|
|
370 |
st.write("[RecursiveCharacterTextSplitter](%s) parameters used:" % url)
|
371 |
st.write(
|
372 |
" chunk_size=%s"
|
|
|
367 |
first_10_tokens = input_ids[:10]
|
368 |
first_10_tokens_text = tokenizer.convert_ids_to_tokens(first_10_tokens)
|
369 |
st.write(first_10_tokens_text)
|
370 |
+
st.write("First 500 tokens:")
|
371 |
+
first_500_tokens = input_ids[:500]
|
372 |
+
first_500_tokens_text = tokenizer.convert_ids_to_tokens(first_500_tokens)
|
373 |
+
st.write(first_500_tokens_text)
|
374 |
st.write("[RecursiveCharacterTextSplitter](%s) parameters used:" % url)
|
375 |
st.write(
|
376 |
" chunk_size=%s"
|