gamingflexer commited on
Commit
1ade467
1 Parent(s): a70510f

Add text splitter to utils.py

Browse files
Files changed (1) hide show
  1. src/utils.py +7 -1
src/utils.py CHANGED
@@ -1,8 +1,14 @@
1
  import json
2
  import uuid
 
 
 
 
 
 
3
 
4
  def generate_uuid():
5
- return str(uuid.uuid4().g
6
 
7
  def check_id_extis_in_json(file_id):
8
  with open('file_ids.json', 'r') as f:
 
1
  import json
2
  import uuid
3
+ from langchain.text_splitter import CharacterTextSplitter
4
+
5
+ text_splitter = CharacterTextSplitter(
6
+ separator="\n",
7
+ chunk_size=3000, chunk_overlap=0
8
+ )
9
 
10
  def generate_uuid():
11
+ return str(uuid.uuid4())
12
 
13
  def check_id_extis_in_json(file_id):
14
  with open('file_ids.json', 'r') as f: