Repository: arabold/docs-mcp-server (ts)

Issue:
Allow exceeding the maximum chunk size if a chunk cannot be split further
Distinguish 3 different chunk size parameters:
- `minChunkSize`: Chunks will be combined until the `minChunkSize` is reached. Chunks under this size will be dropped.
- `preferredChunkSize`: this is the chunk size we're generally targeting
- `maxChunkSize`: No chunk must be larger than maxChunkSize


Pull request description:
Introduce minChunkSize, preferredChunkSize, and maxChunkSize parameters (#1)
This PR addresses #1 by:

- Distinguishing between `minChunkSize`, `preferredChunkSize`, and `maxChunkSize` for chunk splitting
- Refactoring splitters and related logic to use the new parameters
- Updating tests and configuration to reflect new chunk size handling
- Ensuring no chunk exceeds `maxChunkSize` and small chunks are dropped as specified

Closes #1.

Patch:
diff --git a/src/splitter/GreedySplitter.ts b/src/splitter/GreedySplitter.ts
index e97d89a..a3b88d9 100644
--- a/src/splitter/GreedySplitter.ts
+++ b/src/splitter/GreedySplitter.ts
@@ -13,7 +13,7 @@ import type { ContentChunk, DocumentSplitter, SectionContentType } from "./types
 export class GreedySplitter implements DocumentSplitter {
   private baseSplitter: DocumentSplitter;
   private minChunkSize: number;
-  private maxChunkSize: number;
+  private preferredChunkSize: number;
 
   /**
    * Combines a base document splitter with size constraints to produce optimally-sized chunks.
@@ -23,11 +23,11 @@ export class GreedySplitter implements DocumentSplitter {
   constructor(
     baseSplitter: DocumentSplitter,
     minChunkSize: number,
-    maxChunkSize: number,
+    preferredChunkSize: number,
   ) {
     this.baseSplitter = baseSplitter;
     this.minChunkSize = minChunkSize;
-    this.maxChunkSize = maxChunkSize;
+    this.preferredChunkSize = preferredChunkSize;
   }
 
   /**
@@ -101,7 +101,9 @@ export class GreedySplitter implements DocumentSplitter {
     if (!currentChunk) {
       return false;
     }
-    return currentChunk.content.length + nextChunk.content.length > this.maxChunkSize;
+    return (
+      currentChunk.content.length + nextChunk.content.length > this.preferredChunkSize
+    );
   }
 
   /**
diff --git a/src/splitter/SemanticMarkdownSplitter.ts b/src/splitter/SemanticMarkdownSplitter.ts
index 0f2a85d..130e844 100644
--- a/src/splitter/SemanticMarkdownSplitter.ts
+++ b/src/splitter/SemanticMarkdownSplitter.ts
@@ -33,7 +33,7 @@ interface DocumentSection {
  *
  * The splitting process happens in two steps:
  * 1. Split document into sections based on headings (H1-H3 only)
- * 2. Split section content into smaller chunks based on maxChunkSize
+ * 2. Split section content into smaller chunks based on preferredChunkSize
  */
 export class SemanticMarkdownSplitter implements DocumentSplitter {
   private turndownService: TurndownService;
@@ -41,7 +41,10 @@ export class SemanticMarkdownSplitter implements DocumentSplitter {
   public codeSplitter: CodeContentSplitter;
   public tableSplitter: TableContentSplitter;
 
-  constructor(private maxChunkSize: number) {
+  constructor(
+    private preferredChunkSize: number,
+    private maxChunkSize: number,
+  ) {
     this.turndownService = new TurndownService({
       headingStyle: "atx",
       hr: "---",
@@ -83,14 +86,16 @@ export class SemanticMarkdownSplitter implements DocumentSplitter {
       },
     });
 
+    // Text splitter uses preferred chunk size (keeps paragraphs together if possible)
     this.textSplitter = new TextContentSplitter({
-      maxChunkSize: this.maxChunkSize,
+      chunkSize: this.preferredChunkSize,
     });
+    // Code/table splitters use the hard chunk size (avoid splitting unless necessary)
     this.codeSplitter = new CodeContentSplitter({
-      maxChunkSize: this.maxChunkSize,
+      chunkSize: this.maxChunkSize,
     });
     this.tableSplitter = new TableContentSplitter({
-      maxChunkSize: this.maxChunkSize,
+      chunkSize: this.maxChunkSize,
     });
   }
 
diff --git a/src/splitter/splitters/CodeContentSplitter.ts b/src/splitter/splitters/CodeContentSplitter.ts
index b5e7b86..2051c04 100644
--- a/src/splitter/splitters/CodeContentSplitter.ts
+++ b/src/splitter/splitters/CodeContentSplitter.ts
@@ -19,17 +19,17 @@ export class CodeContentSplitter implements ContentSplitter {
     let currentChunkLines: string[] = [];
 
     for (const line of lines) {
-      // Check if a single line with code block markers exceeds maxChunkSize
+      // Check if a single line with code block markers exceeds chunkSize
       const singleLineSize = this.wrap(line, language).length;
-      if (singleLineSize > this.options.maxChunkSize) {
-        throw new MinimumChunkSizeError(singleLineSize, this.options.maxChunkSize);
+      if (singleLineSize > this.options.chunkSize) {
+        throw new MinimumChunkSizeError(singleLineSize, this.options.chunkSize);
       }
 
       currentChunkLines.push(line);
       const newChunkContent = this.wrap(currentChunkLines.join("\n"), language);
       const newChunkSize = newChunkContent.length;
 
-      if (newChunkSize > this.options.maxChunkSize && currentChunkLines.length > 1) {
+      if (newChunkSize > this.options.chunkSize && currentChunkLines.length > 1) {
         // remove last item
         const lastLine = currentChunkLines.pop();
         // wrap content and create chunk
diff --git a/src/splitter/splitters/TableContentSplitter.ts b/src/splitter/splitters/TableContentSplitter.ts
index ebb66ae..a9604ff 100644
--- a/src/splitter/splitters/TableContentSplitter.ts
+++ b/src/splitter/splitters/TableContentSplitter.ts
@@ -32,15 +32,15 @@ export class TableContentSplitter implements ContentSplitter {
     let currentRows: string[] = [];
 
     for (const row of rows) {
-      // Check if a single row with headers exceeds maxChunkSize
+      // Check if a single row with headers exceeds chunkSize
       const singleRowSize = this.wrap(row, headers).length;
-      if (singleRowSize > this.options.maxChunkSize) {
-        throw new MinimumChunkSizeError(singleRowSize, this.options.maxChunkSize);
+      if (singleRowSize > this.options.chunkSize) {
+        throw new MinimumChunkSizeError(singleRowSize, this.options.chunkSize);
       }
 
       const newChunkContent = this.wrap([...currentRows, row].join("\n"), headers);
       const newChunkSize = newChunkContent.length;
-      if (newChunkSize > this.options.maxChunkSize && currentRows.length > 0) {
+      if (newChunkSize > this.options.chunkSize && currentRows.length > 0) {
         // Add current chunk, start new
         chunks.push(this.wrap(currentRows.join("\n"), headers));
         currentRows = [row];
diff --git a/src/splitter/splitters/TextContentSplitter.ts b/src/splitter/splitters/TextContentSplitter.ts
index c937f6e..90e6d6d 100644
--- a/src/splitter/splitters/TextContentSplitter.ts
+++ b/src/splitter/splitters/TextContentSplitter.ts
@@ -19,17 +19,17 @@ export class TextContentSplitter implements ContentSplitter {
   async split(content: string): Promise<string[]> {
     const trimmedContent = fullTrim(content);
 
-    if (trimmedContent.length <= this.options.maxChunkSize) {
+    if (trimmedContent.length <= this.options.chunkSize) {
       return [trimmedContent];
     }
 
-    // Check for unsplittable content (e.g., a single word longer than maxChunkSize)
+    // Check for unsplittable content (e.g., a single word longer than chunkSize)
     const words = trimmedContent.split(/\s+/);
     const longestWord = words.reduce((max, word) =>
       word.length > max.length ? word : max,
     );
-    if (longestWord.length > this.options.maxChunkSize) {
-      throw new MinimumChunkSizeError(longestWord.length, this.options.maxChunkSize);
+    if (longestWord.length > this.options.chunkSize) {
+      throw new MinimumChunkSizeError(longestWord.length, this.options.chunkSize);
     }
 
     // First try splitting by paragraphs (double newlines)
@@ -54,7 +54,7 @@ export class TextContentSplitter implements ContentSplitter {
    * Checks if all chunks are within the maximum size limit
    */
   private areChunksValid(chunks: string[]): boolean {
-    return chunks.every((chunk) => chunk.length <= this.options.maxChunkSize);
+    return chunks.every((chunk) => chunk.length <= this.options.chunkSize);
   }
 
   /**
@@ -86,7 +86,7 @@ export class TextContentSplitter implements ContentSplitter {
    */
   private async splitByWords(text: string): Promise<string[]> {
     const splitter = new RecursiveCharacterTextSplitter({
-      chunkSize: this.options.maxChunkSize,
+      chunkSize: this.options.chunkSize,
       chunkOverlap: 0,
     });
 
@@ -111,10 +111,7 @@ export class TextContentSplitter implements ContentSplitter {
       const currentChunkSize = this.getChunkSize(currentChunk);
       const nextChunkSize = this.getChunkSize(chunk);
 
-      if (
-        currentChunkSize + nextChunkSize + separator.length <=
-        this.options.maxChunkSize
-      ) {
+      if (currentChunkSize + nextChunkSize + separator.length <= this.options.chunkSize) {
         // Merge chunks
         currentChunk = `${currentChunk}${separator}${chunk}`;
       } else {
diff --git a/src/splitter/splitters/types.ts b/src/splitter/splitters/types.ts
index bbd7027..20abe34 100644
--- a/src/splitter/splitters/types.ts
+++ b/src/splitter/splitters/types.ts
@@ -3,7 +3,7 @@
  */
 export interface ContentSplitterOptions {
   /** Maximum characters per chunk */
-  maxChunkSize: number;
+  chunkSize: number;
 }
 
 /**
diff --git a/src/store/DocumentManagementService.ts b/src/store/DocumentManagementService.ts
index 96d5fd8..37eca84 100644
--- a/src/store/DocumentManagementService.ts
+++ b/src/store/DocumentManagementService.ts
@@ -7,6 +7,11 @@ import semver from "semver";
 import { GreedySplitter, SemanticMarkdownSplitter } from "../splitter";
 import type { ContentChunk, DocumentSplitter } from "../splitter/types";
 import { LibraryNotFoundError, VersionNotFoundError } from "../tools";
+import {
+  SPLITTER_MAX_CHUNK_SIZE,
+  SPLITTER_MIN_CHUNK_SIZE,
+  SPLITTER_PREFERRED_CHUNK_SIZE,
+} from "../utils/config";
 import { logger } from "../utils/logger";
 import { getProjectRoot } from "../utils/paths";
 import { DocumentRetrieverService } from "./DocumentRetrieverService";
@@ -77,13 +82,14 @@ export class DocumentManagementService {
     this.store = new DocumentStore(dbPath);
     this.documentRetriever = new DocumentRetrieverService(this.store);
 
-    const minChunkSize = 500;
-    const maxChunkSize = 1500;
-    const semanticSplitter = new SemanticMarkdownSplitter(maxChunkSize);
+    const semanticSplitter = new SemanticMarkdownSplitter(
+      SPLITTER_PREFERRED_CHUNK_SIZE,
+      SPLITTER_MAX_CHUNK_SIZE,
+    );
     const greedySplitter = new GreedySplitter(
       semanticSplitter,
-      minChunkSize,
-      maxChunkSize,
+      SPLITTER_MIN_CHUNK_SIZE,
+      SPLITTER_PREFERRED_CHUNK_SIZE,
     );
 
     this.splitter = greedySplitter;
diff --git a/src/utils/config.ts b/src/utils/config.ts
index 5bef220..20b9966 100644
--- a/src/utils/config.ts
+++ b/src/utils/config.ts
@@ -26,3 +26,10 @@ export const FETCHER_MAX_RETRIES = 6;
  * Base delay in milliseconds for HTTP fetcher retry backoff.
  */
 export const FETCHER_BASE_DELAY = 1000;
+
+/**
+ * Default chunk size settings for splitters
+ */
+export const SPLITTER_MIN_CHUNK_SIZE = 500;
+export const SPLITTER_PREFERRED_CHUNK_SIZE = 1500;
+export const SPLITTER_MAX_CHUNK_SIZE = 5000;


Test patch:
diff --git a/src/splitter/GreedySplitter.test.ts b/src/splitter/GreedySplitter.test.ts
index 26c3249..9a60419 100644
--- a/src/splitter/GreedySplitter.test.ts
+++ b/src/splitter/GreedySplitter.test.ts
@@ -101,7 +101,7 @@ describe("GreedySplitter", () => {
     ]);
   });
 
-  it("should not exceed maxChunkSize", async () => {
+  it("should not exceed preferredChunkSize", async () => {
     const initialChunks: ContentChunk[] = [
       {
         types: ["text"],
@@ -115,7 +115,7 @@ describe("GreedySplitter", () => {
       },
     ];
     const mockSemanticSplitter = createMockSemanticSplitter(initialChunks);
-    const splitter = new GreedySplitter(mockSemanticSplitter, 10, 30); // maxChunkSize = 30
+    const splitter = new GreedySplitter(mockSemanticSplitter, 10, 30); // preferredChunkSize = 30
     const result = await splitter.splitText("Some Markdown");
     expect(result).toEqual([
       {
@@ -384,7 +384,7 @@ Some other text
 `;
 
     // Create a *real* SemanticMarkdownSplitter to get the initial chunks
-    const realSemanticSplitter = new SemanticMarkdownSplitter(200);
+    const realSemanticSplitter = new SemanticMarkdownSplitter(200, 5000);
     const initialChunks = await realSemanticSplitter.splitText(markdown);
 
     const mockSemanticSplitter = createMockSemanticSplitter(initialChunks);
diff --git a/src/splitter/SemanticMarkdownSplitter.test.ts b/src/splitter/SemanticMarkdownSplitter.test.ts
index 0b93976..96d3fd6 100644
--- a/src/splitter/SemanticMarkdownSplitter.test.ts
+++ b/src/splitter/SemanticMarkdownSplitter.test.ts
@@ -5,13 +5,13 @@ vi.mock("../utils/logger");
 
 describe("SemanticMarkdownSplitter", () => {
   it("should handle empty markdown", async () => {
-    const splitter = new SemanticMarkdownSplitter(100);
+    const splitter = new SemanticMarkdownSplitter(100, 5000);
     const result = await splitter.splitText("");
     expect(result).toEqual([]);
   });
 
   it("should handle markdown with no headings", async () => {
-    const splitter = new SemanticMarkdownSplitter(100);
+    const splitter = new SemanticMarkdownSplitter(100, 5000);
     const markdown = "This is some text without any headings.";
     const result = await splitter.splitText(markdown);
 
@@ -28,7 +28,7 @@ describe("SemanticMarkdownSplitter", () => {
   });
 
   it("should correctly split on H1-H6 headings", async () => {
-    const splitter = new SemanticMarkdownSplitter(100);
+    const splitter = new SemanticMarkdownSplitter(100, 5000);
     const markdown = `
 # Chapter 1
 Some text in chapter 1.
@@ -216,7 +216,7 @@ Text in chapter 2.
   });
 
   it("should separate headings, text, code, and tables", async () => {
-    const splitter = new SemanticMarkdownSplitter(100);
+    const splitter = new SemanticMarkdownSplitter(100, 5000);
     const markdown = `
 # Mixed Content Section
 
@@ -271,9 +271,9 @@ console.log('Hello');
   });
 
   it("should correctly split long tables while preserving headers", async () => {
-    const splitter = new SemanticMarkdownSplitter(100);
+    const splitter = new SemanticMarkdownSplitter(10, 100);
 
-    // Create a table with many rows that will exceed maxChunkSize
+    // Create a table with many rows that will exceed chunkSize
     const tableRows = Array.from(
       { length: 20 },
       (_, i) => `| ${i + 1} | This is row ${i + 1} | ${(i + 1) * 100} |`,
@@ -307,9 +307,9 @@ ${tableRows}
   });
 
   it("should correctly split long code blocks while preserving language", async () => {
-    const splitter = new SemanticMarkdownSplitter(100);
+    const splitter = new SemanticMarkdownSplitter(10, 100);
 
-    // Create a long code block that will exceed maxChunkSize
+    // Create a long code block that will exceed chunkSize
     const codeLines = Array.from(
       { length: 20 },
       (_, i) =>
@@ -342,7 +342,7 @@ ${codeLines}
   });
 
   it("should handle tables that cannot be split semantically by using character-based splitting", async () => {
-    const splitter = new SemanticMarkdownSplitter(20);
+    const splitter = new SemanticMarkdownSplitter(20, 20);
     const markdown = `
 | Header1 | Header2 |
 |---------|---------|
@@ -359,7 +359,7 @@ ${codeLines}
   });
 
   it("should handle code blocks that cannot be split semantically by using character-based splitting", async () => {
-    const splitter = new SemanticMarkdownSplitter(20);
+    const splitter = new SemanticMarkdownSplitter(20, 20);
     const markdown = "```javascript\nconst x = 1;\n```";
 
     // Should not throw an error anymore
diff --git a/src/splitter/splitters/CodeContentSplitter.test.ts b/src/splitter/splitters/CodeContentSplitter.test.ts
index c4246dc..e7ee4bf 100644
--- a/src/splitter/splitters/CodeContentSplitter.test.ts
+++ b/src/splitter/splitters/CodeContentSplitter.test.ts
@@ -6,7 +6,7 @@ vi.mock("../../utils/logger");
 
 describe("CodeContentSplitter", () => {
   const options = {
-    maxChunkSize: 100,
+    chunkSize: 100,
   } satisfies ContentSplitterOptions;
   const splitter = new CodeContentSplitter(options);
 
@@ -38,7 +38,7 @@ const y = 2;`;
     const chunks = await splitter.split(markdown);
     expect(chunks.length).toBeGreaterThan(1);
     for (const chunk of chunks) {
-      expect(chunk.length).toBeLessThanOrEqual(options.maxChunkSize);
+      expect(chunk.length).toBeLessThanOrEqual(options.chunkSize);
       expect(chunk.startsWith("```javascript\n")).toBe(true);
       expect(chunk.endsWith("\n```")).toBe(true);
     }
diff --git a/src/splitter/splitters/TableContentSplitter.test.ts b/src/splitter/splitters/TableContentSplitter.test.ts
index e4edcc0..d7adfb6 100644
--- a/src/splitter/splitters/TableContentSplitter.test.ts
+++ b/src/splitter/splitters/TableContentSplitter.test.ts
@@ -7,7 +7,7 @@ vi.mock("../../utils/logger");
 
 describe("TableContentSplitter", () => {
   const options = {
-    maxChunkSize: 100,
+    chunkSize: 100,
   } satisfies ContentSplitterOptions;
   const splitter = new TableContentSplitter(options);
 
@@ -27,7 +27,7 @@ describe("TableContentSplitter", () => {
   });
 
   it("should split large tables by rows", async () => {
-    // Create a large table that *might* exceed maxChunkSize, depending on header length
+    // Create a large table that *might* exceed chunkSize, depending on header length
     const rows = Array(20)
       .fill(0)
       .map((_, i) => `| Data ${i}A | Data ${i}B |`);
@@ -44,9 +44,9 @@ ${rows.join("\n")}`;
     }
   });
 
-  it("should throw MinimumChunkSizeError if single row with headers exceeds maxChunkSize", async () => {
+  it("should throw MinimumChunkSizeError if single row with headers exceeds chunkSize", async () => {
     const splitter = new TableContentSplitter({
-      maxChunkSize: 50, // Small size for testing
+      chunkSize: 50, // Small size for testing
     });
     const table = `| Header A | Header B | Header C |
 |----------|-----------|-----------|
diff --git a/src/splitter/splitters/TextContentSplitter.test.ts b/src/splitter/splitters/TextContentSplitter.test.ts
index ba45a20..09b523c 100644
--- a/src/splitter/splitters/TextContentSplitter.test.ts
+++ b/src/splitter/splitters/TextContentSplitter.test.ts
@@ -6,7 +6,7 @@ vi.mock("../../utils/logger");
 
 describe("TextContentSplitter", () => {
   const options = {
-    maxChunkSize: 100,
+    chunkSize: 100,
   } satisfies ContentSplitterOptions;
   const splitter = new TextContentSplitter(options);
 
@@ -26,7 +26,7 @@ Third paragraph to complete the example.`;
   });
 
   it("should fall back to line breaks when paragraphs too large", async () => {
-    // Create a paragraph larger than maxChunkSize
+    // Create a paragraph larger than preferredChunkSize
     const longParagraph = Array(5)
       .fill("This is a very long line of text that should be split.")
       .join(" ");
@@ -41,7 +41,7 @@ And line four finishes it.`;
     // Should split into multiple chunks at line boundaries
     expect(chunks.length).toBeGreaterThan(1);
     for (const chunk of chunks) {
-      expect(chunk.length).toBeLessThanOrEqual(options.maxChunkSize);
+      expect(chunk.length).toBeLessThanOrEqual(options.chunkSize);
     }
   });
 
@@ -54,7 +54,7 @@ And line four finishes it.`;
     // Small consecutive lines should be merged
     expect(chunks.length).toBeLessThan(6); // Less than total number of lines
     for (const chunk of chunks) {
-      expect(chunk.length).toBeLessThanOrEqual(options.maxChunkSize);
+      expect(chunk.length).toBeLessThanOrEqual(options.chunkSize);
     }
   });
 
@@ -70,7 +70,7 @@ And line four finishes it.`;
 
   it("should split words as last resort", async () => {
     const splitter = new TextContentSplitter({
-      maxChunkSize: 20, // Very small for testing word splitting
+      chunkSize: 20, // Very small for testing word splitting
     });
 
     const text =