nbaldwin commited on
Commit
fc0b4d8
·
1 Parent(s): ef61dcf

added seperator arg

Browse files
Files changed (2) hide show
  1. ChromaDBFlow.py +3 -1
  2. ChromaDBFlow.yaml +1 -0
ChromaDBFlow.py CHANGED
@@ -42,6 +42,7 @@ class ChromaDBFlow(AtomicFlow):
42
  - `filter` (str): The filter to apply to the documents. Default: null
43
  - `paths_to_data` (List[str]): The paths to the data to store in the database at instantiation. Default: []
44
  - `chunk_size` (int): The size of the chunks to split the documents into. Default: 700
 
45
  - `chunk_overlap` (int): The overlap between the chunks. Default: 0
46
  - `persist_directory` (str): The directory to persist the database. Default: "./demo_db_dir"
47
 
@@ -133,7 +134,8 @@ class ChromaDBFlow(AtomicFlow):
133
  full_docs = []
134
  text_splitter = CharacterTextSplitter(
135
  chunk_size=self.flow_config["chunk_size"],
136
- chunk_overlap=self.flow_config["chunk_overlap"]
 
137
  )
138
 
139
  for path in self.flow_config["paths_to_data"]:
 
42
  - `filter` (str): The filter to apply to the documents. Default: null
43
  - `paths_to_data` (List[str]): The paths to the data to store in the database at instantiation. Default: []
44
  - `chunk_size` (int): The size of the chunks to split the documents into. Default: 700
45
+ - `seperator` (str): The separator to use to split the documents. Default: "\n"
46
  - `chunk_overlap` (int): The overlap between the chunks. Default: 0
47
  - `persist_directory` (str): The directory to persist the database. Default: "./demo_db_dir"
48
 
 
134
  full_docs = []
135
  text_splitter = CharacterTextSplitter(
136
  chunk_size=self.flow_config["chunk_size"],
137
+ chunk_overlap=self.flow_config["chunk_overlap"],
138
+ separator=self.flow_config["separator"]
139
  )
140
 
141
  for path in self.flow_config["paths_to_data"]:
ChromaDBFlow.yaml CHANGED
@@ -16,4 +16,5 @@ output_interface:
16
  paths_to_data: []
17
  chunk_size: 700
18
  chunk_overlap: 0
 
19
  persist_directory: ./demo_db_dir
 
16
  paths_to_data: []
17
  chunk_size: 700
18
  chunk_overlap: 0
19
+ separator: "\n"
20
  persist_directory: ./demo_db_dir