Spaces:
Sleeping
Sleeping
| """Unit tests for src/chunker.py — the one rule that matters: statements start | |
| chunks, Proof/Solution glue to the statement before them.""" | |
| from src.chunker import ( | |
| CHILD_MAX_TOK, CHARS_PER_TOKEN, Chunk, chunk_markdown, est_tokens, | |
| split_children, _paragraphs, | |
| ) | |
| def _mk(md: str): | |
| return chunk_markdown(md, doc_id="doc", source="test.md") | |
| def test_statement_starts_new_chunk_and_proof_glues(): | |
| md = ( | |
| "# 6.2 Variance of Discrete Random Variables\n\n" | |
| "Theorem 6.9 The variance is additive for independent variables.\n\n" | |
| "**Proof.** Expand the square and use linearity.\n\n" | |
| "Example 6.1 Compute the variance of one die roll.\n" | |
| ) | |
| chunks = _mk(md) | |
| assert len(chunks) == 2 | |
| assert chunks[0].type == "theorem" | |
| assert "Proof." in chunks[0].text # glued, never its own chunk | |
| assert chunks[1].type == "example" | |
| def test_chapter_pinned_by_numbering_not_heading_depth(): | |
| md = ( | |
| "# Chapter 6\n\nIntro prose.\n\n" | |
| "# 6.2 Variance\n\nSection prose.\n\n" | |
| "# Examples\n\nExample 6.5 Something.\n" | |
| ) | |
| chunks = _mk(md) | |
| assert all(c.chapter == "Chapter 6: Expected Value and Variance" for c in chunks) | |
| # "Examples" is a generic header: it must NOT overwrite the numbered section. | |
| assert chunks[-1].section == "6.2 Variance" | |
| def test_front_matter_does_not_inherit_chapter(): | |
| md = "# Chapter 1\n\nReal content.\n\n# Index\n\nbinomial, 45\n" | |
| chunks = _mk(md) | |
| assert chunks[0].chapter.startswith("Chapter 1") | |
| assert chunks[-1].chapter == "" # store.retrieval_units drops these | |
| def test_unnumbered_topic_header_refines_section_only(): | |
| md = "# Chapter 2\n\nA.\n\n## Standard Deviation\n\nB.\n" | |
| chunks = _mk(md) | |
| assert chunks[-1].chapter.startswith("Chapter 2") | |
| assert chunks[-1].section == "Standard Deviation" | |
| def test_statement_marker_inside_display_math_does_not_split(): | |
| md = "Some setup text.\n$$\nExample = 5\n$$\nmore of the same block.\n" | |
| chunks = _mk(md) | |
| assert len(chunks) == 1 | |
| def test_small_parent_has_no_children(): | |
| p = Chunk(id="d::p1", doc_id="d", source="s", type="text", chapter="", section="", | |
| text="short", n_tokens=est_tokens("short")) | |
| assert split_children(p) == [] | |
| def test_large_parent_splits_with_parent_id_and_full_coverage(): | |
| paras = [f"Paragraph {i} " + "word " * 60 for i in range(10)] | |
| text = "\n\n".join(paras) | |
| p = Chunk(id="d::p1", doc_id="d", source="s", type="text", chapter="C", section="S", | |
| text=text, n_tokens=est_tokens(text)) | |
| assert p.n_tokens > CHILD_MAX_TOK | |
| kids = split_children(p) | |
| assert len(kids) >= 2 | |
| assert all(k.parent_id == "d::p1" and not k.is_parent for k in kids) | |
| assert all(k.chapter == "C" and k.section == "S" for k in kids) | |
| joined = "\n".join(k.text for k in kids) | |
| assert all(f"Paragraph {i}" in joined for i in range(10)) # nothing dropped | |
| def test_paragraphs_keep_display_blocks_intact(): | |
| text = "before\n\n$$\na = 1\n\nb = 2\n$$\n\nafter" | |
| paras = _paragraphs(text) | |
| assert len(paras) == 3 | |
| assert "a = 1" in paras[1] and "b = 2" in paras[1] # blank line inside $$ kept | |
| def test_est_tokens_floor(): | |
| assert est_tokens("") == 1 | |
| assert est_tokens("x" * (CHARS_PER_TOKEN * 10)) == 10 | |