Changed ingestion Logic

ingest.py

@@ -25,28 +25,28 @@ CHUNKING_TECHNIQUES = [
     {
         "name": "sentence",
         "description": "Sentence-level chunking - respects sentence boundaries (NLTK)",
-        "chunk_size": 1000,
+        "chunk_size": 2400,
         "chunk_overlap": 100,
         "kwargs": {},
     },
     {
         "name": "paragraph",
         "description": "Paragraph-level chunking - uses natural paragraph breaks",
-        "chunk_size": 1000,
+        "chunk_size": 4000,
         "chunk_overlap": 100,
         "kwargs": {"separator": "\n\n"},  # Split on paragraph breaks
     },
     {
         "name": "semantic",
         "description": "Semantic chunking - splits where topic/meaning shifts (embedding similarity)",
-        "chunk_size": 1000,
+        "chunk_size": 2000,
         "chunk_overlap": 100,
         "kwargs": {"breakpoint_threshold_type": "percentile", "breakpoint_threshold_amount": 70},
     },
     {
         "name": "recursive",
         "description": "Recursive chunking - hierarchical splitting (paragraphs → sentences → words → chars)",
-        "chunk_size": 1000,
+        "chunk_size": 2000,
         "chunk_overlap": 100,
         "kwargs": {"separators": ["\n\n", "\n", ". ", "! ", "? ", "; ", ", ", " ", ""], "keep_separator": True},
     },

retriever/processor.py

@@ -32,8 +32,10 @@ class ChunkProcessor:
         - "fixed":     Character-based, may split mid-sentence
         - "recursive": Recursive character splitting with hierarchical separators
         - "character": Character-based splitting on paragraph boundaries
+        - "paragraph": Paragraph-level splitting on \\n\\n boundaries
         - "sentence":  Sliding window over NLTK sentences
         - "semantic":  Embedding-based semantic chunking
+        - "page":      Page-level splitting on page markers
         """
         if technique == "fixed":
             return CharacterTextSplitter(
@@ -62,6 +64,16 @@ class ChunkProcessor:
                 is_separator_regex=False
             )
 
+        elif technique == "paragraph":
+            # Paragraph-level chunking using paragraph breaks
+            return CharacterTextSplitter(
+                separator=kwargs.get('separator', "\n\n"),
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                length_function=len,
+                is_separator_regex=False
+            )
+
         elif technique == "sentence":
             # sentence-level chunking using NLTK
             return NLTKTextSplitter(
@@ -78,8 +90,18 @@ class ChunkProcessor:
                 breakpoint_threshold_amount=kwargs.get('breakpoint_threshold_amount', 70)
             )
 
+        elif technique == "page":
+            # Page-level chunking using page markers
+            return CharacterTextSplitter(
+                separator=kwargs.get('separator', "--- Page"),
+                chunk_size=chunk_size,
+                chunk_overlap=chunk_overlap,
+                length_function=len,
+                is_separator_regex=False
+            )
+
         else:
-            raise ValueError(f"Technique '{technique}' is not supported. Choose from: fixed, recursive, character, sentence, semantic")
+            raise ValueError(f"Technique '{technique}' is not supported. Choose from: fixed, recursive, character, paragraph, sentence, semantic, page")
 
     # ------------------------------------------------------------------
     # Processing