chunk_visualizer

Running

m-ric HF staff commited on Feb 15

Commit

6334a1e

•

1 Parent(s): 02271b4

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -3,9 +3,10 @@ from langchain.text_splitter import (
     CharacterTextSplitter,
     RecursiveCharacterTextSplitter,
 )
 LABEL_TEXTSPLITTER = "LangChain's CharacterTextSplitter"
-LABEL_RECURSIVE = "Langchain's RecursiveCharacterTextSplitter"
 bert_tokenizer = AutoTokenizer.from_pretrained('google-bert/bert-base-uncased')

     CharacterTextSplitter,
     RecursiveCharacterTextSplitter,
 )
+from transformers import AutoTokenizer
 LABEL_TEXTSPLITTER = "LangChain's CharacterTextSplitter"
+LABEL_RECURSIVE = "LangChain's RecursiveCharacterTextSplitter"
 bert_tokenizer = AutoTokenizer.from_pretrained('google-bert/bert-base-uncased')