Spaces:
Runtime error
Runtime error
gamingflexer
commited on
Commit
•
1ade467
1
Parent(s):
a70510f
Add text splitter to utils.py
Browse files- src/utils.py +7 -1
src/utils.py
CHANGED
@@ -1,8 +1,14 @@
|
|
1 |
import json
|
2 |
import uuid
|
|
|
|
|
|
|
|
|
|
|
|
|
3 |
|
4 |
def generate_uuid():
|
5 |
-
return str(uuid.uuid4()
|
6 |
|
7 |
def check_id_extis_in_json(file_id):
|
8 |
with open('file_ids.json', 'r') as f:
|
|
|
1 |
import json
|
2 |
import uuid
|
3 |
+
from langchain.text_splitter import CharacterTextSplitter
|
4 |
+
|
5 |
+
text_splitter = CharacterTextSplitter(
|
6 |
+
separator="\n",
|
7 |
+
chunk_size=3000, chunk_overlap=0
|
8 |
+
)
|
9 |
|
10 |
def generate_uuid():
|
11 |
+
return str(uuid.uuid4())
|
12 |
|
13 |
def check_id_extis_in_json(file_id):
|
14 |
with open('file_ids.json', 'r') as f:
|