feat(chunks): avoid small chunks

anyqa/build_index.py

@@ -10,6 +10,7 @@ from .config import get_sources
 from .embeddings import EMBEDDING_MODEL_NAME
 from .vectorstore import PERSIST_DIRECTORY, get_vectorstore
 
+MIN_CHUNK_SIZE = 100
 
 def load_data():
     print("Loading data...")
@@ -39,6 +40,8 @@ def parse_data():
         doc_chunks = text_splitter.split_documents(pages)
 
         for chunk in doc_chunks:
+            if len(chunk.page_content) < MIN_CHUNK_SIZE:
+                continue
             chunk.metadata["name"] = source["name"]
             chunk.metadata["domain"] = source["domain"]
             url = source.get("url", None)