Repository: instructlab/instructlab

Issue:
Knowledge documents with very long lines do not get chunked properly
We only chunk knowledge documents on newlines at https://github.com/instructlab/instructlab/blob/fd1be3f2ae2bb4ea90822566de58651122de1537/src/instructlab/utils.py#L259

In testing some more real-world use-cases, I'm running across knowledge documents that have some very long lines. These documents are converted to markdown from other systems, and in some cases have stacktraces, very long absolute paths with filenames, and so on that leads to very long lines. When a document chunk with one of these long lines gets selected, we still blow up the context window with errors like `Exception: Requested tokens (4488) exceed context window of 4096`

Should we chunk on something besides newline tokens? Should we check our chunk sizes after chunking and drop any chunks that are too large with a warning? Do we not change anything and just document a maximum line length?

I think the cleanest solution is to check the chunk sizes after we've split them and drop any chunks that are above our threshold with a warning that the file contains very long lines that will get skipped or something of the sort. I'm happy to hear thoughts from others about the best way to handle this.


Discussion:
Actually, the best thing may be to enable splitting on spaces as well. The splits are in priority order, so we'll only split on spaces if we couldn't create small enough chunks by splitting on paragraphs or newlines.



Patch:
diff --git a/src/instructlab/utils.py b/src/instructlab/utils.py
index 44f759a150..190946eb49 100644
--- a/src/instructlab/utils.py
+++ b/src/instructlab/utils.py
@@ -34,6 +34,8 @@
     max: 120
 """
 
+DEFAULT_CHUNK_OVERLAP = 100
+
 logging.basicConfig(
     level=logging.INFO,
     format="%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s",
@@ -237,6 +239,14 @@ def git_clone_checkout(
     return repo
 
 
+def num_tokens_from_words(num_words) -> int:
+    return int(num_words * 1.3)  # 1 word ~ 1.3 token
+
+
+def num_chars_from_tokens(num_tokens) -> int:
+    return int(num_tokens * 4)  # 1 token ~ 4 English character
+
+
 def chunk_document(documents: List, server_ctx_size, chunk_word_count) -> List[str]:
     """
     Iterates over the documents and splits them into chunks based on the word count provided by the user.
@@ -247,7 +257,7 @@ def chunk_document(documents: List, server_ctx_size, chunk_word_count) -> List[s
     Returns:
          List[str]: List of chunked documents.
     """
-    no_tokens_per_doc = int(chunk_word_count * 1.3)  # 1 word ~ 1.3 token
+    no_tokens_per_doc = num_tokens_from_words(chunk_word_count)
     if no_tokens_per_doc > int(server_ctx_size - 1024):
         raise ValueError(
             "Error: {}".format(
@@ -258,9 +268,9 @@ def chunk_document(documents: List, server_ctx_size, chunk_word_count) -> List[s
         )
     content = []
     text_splitter = RecursiveCharacterTextSplitter(
-        separators=["\n\n", "\n"],
-        chunk_size=int(no_tokens_per_doc * 4),  # 1 token ~ 4 English character
-        chunk_overlap=100,
+        separators=["\n\n", "\n", " "],
+        chunk_size=num_chars_from_tokens(no_tokens_per_doc),
+        chunk_overlap=DEFAULT_CHUNK_OVERLAP,
     )
 
     for docs in documents:


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 1dc0d6b00b..2dd072aafb 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -43,6 +43,20 @@ def test_chunk_docs_chunk_overlap_error(self):
             f"{exc.exception}",
         )
 
+    def test_chunk_docs_long_lines(self):
+        chunk_words = 50
+        chunks = utils.chunk_document(
+            documents=testdata.long_line_documents,
+            chunk_word_count=chunk_words,
+            server_ctx_size=4096,
+        )
+        max_tokens = utils.num_tokens_from_words(chunk_words)
+        max_chars = utils.num_chars_from_tokens(max_tokens)
+        max_chars += utils.DEFAULT_CHUNK_OVERLAP  # add in the chunk overlap
+        max_chars += 50  # and a bit extra for some really long words
+        for chunk in chunks:
+            assert len(chunk) <= max_chars
+
     @patch(
         "instructlab.utils.git_clone_checkout",
         return_value=Mock(spec=git.Repo, working_dir="tests/testdata/temp_repo"),
diff --git a/tests/testdata/testdata.py b/tests/testdata/testdata.py
index f728a44e6d..24364564a1 100644
--- a/tests/testdata/testdata.py
+++ b/tests/testdata/testdata.py
@@ -11,6 +11,19 @@
       called Gettier cases that provoked alternative definitions."""
 ]
 
+long_line_documents = [
+    (
+        "Knowledge is an awareness of facts, a familiarity with individuals and situations,"
+        "or a practical skill. Knowledge of facts, also called propositional knowledge, is often characterized "
+        "as true belief that is distinct from opinion or guesswork by virtue of justification. "
+        "While there is wide agreement among philosophers that propositional knowledge is a form of true belief, "
+        "many controversies focus on justification. This includes questions like how to understand justification, "
+        "whether it is needed at all, and whether something else besides it is needed. "
+        "These controversies intensified in the latter half of the 20th century due to a series of thought experiments "
+        "called Gettier cases that provoked alternative definitions."
+    )
+]
+
 knowledge_seed_instruction = [
     {
         "instruction": "What is this knowledge about?",